02:06:18.252 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
02:06:18.264 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskQ73J9W/test-tmp
02:06:18.267 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
02:06:18.270 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskQ73J9W/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
02:06:18.538 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
02:06:18.678 [DEBUG - main] (MiniKuduCluster.java:195) Response:
02:06:18.680 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
02:06:18.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:18.683401 22419 mini_oidc.cc:152] Starting JWKS server
02:06:18.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:18.687347 22419 webserver.cc:460] Webserver started at http://127.0.0.1:41051/ using document root <none> and password file <none>
02:06:18.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:18.687888 22419 mini_oidc.cc:189] Starting OIDC Discovery server
02:06:18.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:18.690217 22419 webserver.cc:460] Webserver started at http://127.0.0.1:43081/ using document root <none> and password file <none>
02:06:18.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-26T02:06:18Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
02:06:18.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-26T02:06:18Z Disabled control of system clock
02:06:18.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:18.738140 22419 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQ73J9W/build/tsan/bin/kudu
02:06:18.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQ73J9W/build/tsan/bin/kudu
02:06:18.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/master-0/wal
02:06:18.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/master-0/data
02:06:18.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:06:18.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:06:18.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:06:18.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:18.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:18.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:18.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/master-0/logs
02:06:18.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/master-0/data/info.pb
02:06:18.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:18.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:06:18.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:06:18.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:06:18.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:06:18.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:18.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:18.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:18.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:06:18.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:06:18.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:06:18.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:06:18.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.228.254:43509
02:06:18.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.228.254
02:06:18.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:06:18.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.228.212:41215
02:06:18.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:18.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:18.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:18.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:06:18.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.228.254:43509,127.21.228.253:35519,127.21.228.252:39791
02:06:18.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:06:18.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41051/jwks.json with env {}
02:06:19.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:19.030719 22431 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:06:19.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:19.031378 22431 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:06:19.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:19.032050 22431 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:06:19.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:19.062744 22431 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:06:19.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:19.063094 22431 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:06:19.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:19.063410 22431 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:06:19.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:19.063753 22431 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:06:19.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:19.064098 22431 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:06:19.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:19.064417 22431 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41051/jwks.json
02:06:19.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:19.099375 22431 master_runner.cc:386] Master server non-default flags:
02:06:19.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:19.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.228.212:41215
02:06:19.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:19.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:19.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/master-0/data
02:06:19.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/master-0/wal
02:06:19.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:06:19.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.228.254:43509,127.21.228.253:35519,127.21.228.252:39791
02:06:19.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:19.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:19.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:06:19.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.228.254:43509
02:06:19.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:06:19.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:06:19.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:06:19.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41051/jwks.json
02:06:19.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:19.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:19.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/master-0/data/info.pb
02:06:19.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.228.254
02:06:19.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:06:19.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:06:19.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:19.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:06:19.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:06:19.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:19.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/master-0/logs
02:06:19.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:19.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:06:19.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:06:19.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:06:19.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:06:19.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
02:06:19.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:06:19.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 00:58:27 UTC on ccycloud.quasar-hahlai.root.comops.site
02:06:19.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4673502
02:06:19.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:06:19.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:19.101009 22431 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:06:19.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:19.103124 22431 file_cache.cc:493] Constructed file cache file cache with capacity 419430
02:06:19.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:19.122571 22438 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:20.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:19.122627 22440 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:20.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:19.124294 22437 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:20.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:20.342604 22439 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
02:06:20.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:20.342751 22431 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:06:20.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:20.346958 22431 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:06:20.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:20.349466 22431 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:06:20.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:20.350854 22431 hybrid_clock.cc:648] HybridClock initialized: now 1790388380350813 us; error 48 us; skew 500 ppm
02:06:20.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:20.351739 22431 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:06:20.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:20.363406 22431 webserver.cc:460] Webserver started at http://127.21.228.254:40939/ using document root <none> and password file <none>
02:06:20.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:20.364573 22431 fs_manager.cc:359] Metadata directory not provided
02:06:20.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:20.364826 22431 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:06:20.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:20.365373 22431 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
02:06:20.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:20.370363 22431 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/master-0/data/instance:
02:06:20.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "723fe02fa81b44b08c4b52529d34d100"
02:06:20.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 02:06:20 on dist-test-slave-nz2j"
02:06:20.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:20.371747 22431 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/master-0/wal/instance:
02:06:20.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "723fe02fa81b44b08c4b52529d34d100"
02:06:20.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 02:06:20 on dist-test-slave-nz2j"
02:06:20.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:20.379674 22431 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.001s sys 0.004s
02:06:20.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:20.385368 22447 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:06:20.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:20.386539 22431 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
02:06:20.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:20.386910 22431 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/master-0/data,/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/master-0/wal
02:06:20.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "723fe02fa81b44b08c4b52529d34d100"
02:06:20.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 02:06:20 on dist-test-slave-nz2j"
02:06:20.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:20.387311 22431 fs_report.cc:383] FS layout report
02:06:20.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:06:20.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/master-0/wal
02:06:20.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/master-0/wal
02:06:20.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/master-0/data/data
02:06:20.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:06:20.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:06:20.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:06:20.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:06:20.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:06:20.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:06:20.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:06:20.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:06:20.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:06:20.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:06:20.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:06:20.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:20.428092 22431 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
02:06:20.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:20.429630 22431 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:06:20.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:20.430141 22431 kserver.cc:163] Server-wide thread pool size limit: 3276
02:06:20.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:20.504029 22431 rpc_server.cc:307] RPC server started. Bound to: 127.21.228.254:43509
02:06:20.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:20.504117 22498 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.21.228.254:43509 every 8 connection(s)
02:06:20.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:20.506901 22431 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/master-0/data/info.pb
02:06:20.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:20.512264 22499 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:06:20.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:20.517693 22419 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQ73J9W/build/tsan/bin/kudu as pid 22431
02:06:20.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:20.518122 22419 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/master-0/wal/instance
02:06:20.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:20.522190 22419 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQ73J9W/build/tsan/bin/kudu
02:06:20.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQ73J9W/build/tsan/bin/kudu
02:06:20.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/master-1/wal
02:06:20.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/master-1/data
02:06:20.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:06:20.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:06:20.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:06:20.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:20.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:20.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:20.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/master-1/logs
02:06:20.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/master-1/data/info.pb
02:06:20.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:20.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:06:20.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:06:20.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:06:20.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:06:20.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:20.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:20.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:20.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:06:20.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:06:20.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:06:20.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:06:20.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.228.253:35519
02:06:20.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.228.253
02:06:20.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:06:20.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.228.212:41215
02:06:20.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:20.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:20.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:20.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:06:20.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.228.254:43509,127.21.228.253:35519,127.21.228.252:39791
02:06:20.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:06:20.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41051/jwks.json with env {}
02:06:20.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:20.525209 22499 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.228.254" port: 43509 } has no permanent_uuid. Determining permanent_uuid...
02:06:20.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:20.555590 22499 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.228.253" port: 35519 } has no permanent_uuid. Determining permanent_uuid...
02:06:20.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:20.558177 22450 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.21.228.253:35519: connect: Connection refused (error 111)
02:06:20.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:20.562182 22499 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.228.253:35519: Network error: Client connection negotiation failed: client connection to 127.21.228.253:35519: connect: Connection refused (error 111)
02:06:20.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:20.611692 22499 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.228.253" port: 35519 } attempt: 1
02:06:20.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:20.615442 22499 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.228.253:35519: Network error: Client connection negotiation failed: client connection to 127.21.228.253:35519: connect: Connection refused (error 111)
02:06:20.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:20.683902 22499 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.228.253" port: 35519 } attempt: 2
02:06:20.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:20.688365 22499 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.228.253:35519: Network error: Client connection negotiation failed: client connection to 127.21.228.253:35519: connect: Connection refused (error 111)
02:06:20.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:20.779798 22499 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.228.253" port: 35519 } attempt: 3
02:06:20.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:20.784142 22499 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.228.253:35519: Network error: Client connection negotiation failed: client connection to 127.21.228.253:35519: connect: Connection refused (error 111)
02:06:20.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:20.827266 22501 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:06:20.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:20.827967 22501 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:06:20.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:20.828758 22501 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:06:20.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:20.860394 22501 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:06:20.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:20.860651 22501 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:06:20.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:20.860837 22501 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:06:20.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:20.861020 22501 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:06:20.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:20.861291 22501 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:06:20.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:20.861526 22501 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41051/jwks.json
02:06:20.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:20.895651 22501 master_runner.cc:386] Master server non-default flags:
02:06:20.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:20.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.228.212:41215
02:06:20.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:20.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:20.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/master-1/data
02:06:20.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/master-1/wal
02:06:20.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:06:20.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.228.254:43509,127.21.228.253:35519,127.21.228.252:39791
02:06:20.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:20.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:20.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:06:20.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.228.253:35519
02:06:20.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:06:20.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:06:20.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:06:20.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41051/jwks.json
02:06:20.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:20.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:20.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/master-1/data/info.pb
02:06:20.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.228.253
02:06:20.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:06:20.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:06:20.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:20.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:06:20.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:06:20.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:20.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/master-1/logs
02:06:20.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:20.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:06:20.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:06:20.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:06:20.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:06:20.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
02:06:20.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:06:20.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 00:58:27 UTC on ccycloud.quasar-hahlai.root.comops.site
02:06:20.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4673502
02:06:20.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:06:20.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:20.896970 22501 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:06:20.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:20.899073 22501 file_cache.cc:493] Constructed file cache file cache with capacity 419430
02:06:20.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:20.917783 22511 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:20.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:20.927608 22499 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.228.253" port: 35519 } attempt: 4
02:06:20.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:20.931286 22499 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.228.253:35519: Network error: Client connection negotiation failed: client connection to 127.21.228.253:35519: connect: Connection refused (error 111)
02:06:21.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:21.230715 22499 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.228.253" port: 35519 } attempt: 5
02:06:21.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:21.235395 22499 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.228.253:35519: Network error: Client connection negotiation failed: client connection to 127.21.228.253:35519: connect: Connection refused (error 111)
02:06:21.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:21.783039 22499 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.228.253" port: 35519 } attempt: 6
02:06:21.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:21.792644 22499 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.228.253:35519: Network error: Client connection negotiation failed: client connection to 127.21.228.253:35519: connect: Connection refused (error 111)
02:06:22.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:20.917810 22510 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:22.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:20.919431 22501 server_base.cc:1034] running on GCE node
02:06:22.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:20.917800 22513 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:22.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:22.144915 22501 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:06:22.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:22.148372 22501 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:06:22.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:22.149782 22501 hybrid_clock.cc:648] HybridClock initialized: now 1790388382149750 us; error 56 us; skew 500 ppm
02:06:22.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:22.150552 22501 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:06:22.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:22.157260 22501 webserver.cc:460] Webserver started at http://127.21.228.253:35173/ using document root <none> and password file <none>
02:06:22.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:22.158182 22501 fs_manager.cc:359] Metadata directory not provided
02:06:22.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:22.158357 22501 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:06:22.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:22.158722 22501 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
02:06:22.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:22.163354 22501 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/master-1/data/instance:
02:06:22.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7891b700a4394fcfb8996911f72f37f0"
02:06:22.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 02:06:22 on dist-test-slave-nz2j"
02:06:22.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:22.164477 22501 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/master-1/wal/instance:
02:06:22.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7891b700a4394fcfb8996911f72f37f0"
02:06:22.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 02:06:22 on dist-test-slave-nz2j"
02:06:22.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:22.171128 22501 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.005s sys 0.000s
02:06:22.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:22.176770 22522 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:06:22.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:22.177791 22501 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
02:06:22.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:22.178128 22501 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/master-1/data,/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/master-1/wal
02:06:22.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7891b700a4394fcfb8996911f72f37f0"
02:06:22.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 02:06:22 on dist-test-slave-nz2j"
02:06:22.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:22.178484 22501 fs_report.cc:383] FS layout report
02:06:22.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:06:22.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/master-1/wal
02:06:22.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/master-1/wal
02:06:22.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/master-1/data/data
02:06:22.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:06:22.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:06:22.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:06:22.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:06:22.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:06:22.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:06:22.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:06:22.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:06:22.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:06:22.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:06:22.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:06:22.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:22.214618 22501 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
02:06:22.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:22.216075 22501 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:06:22.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:22.216629 22501 kserver.cc:163] Server-wide thread pool size limit: 3276
02:06:22.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:22.288717 22501 rpc_server.cc:307] RPC server started. Bound to: 127.21.228.253:35519
02:06:22.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:22.288800 22573 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.21.228.253:35519 every 8 connection(s)
02:06:22.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:22.291566 22501 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/master-1/data/info.pb
02:06:22.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:22.292316 22419 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQ73J9W/build/tsan/bin/kudu as pid 22501
02:06:22.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:22.292675 22419 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/master-1/wal/instance
02:06:22.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:22.295430 22419 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQ73J9W/build/tsan/bin/kudu
02:06:22.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQ73J9W/build/tsan/bin/kudu
02:06:22.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/master-2/wal
02:06:22.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/master-2/data
02:06:22.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:06:22.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:06:22.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:06:22.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:22.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:22.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:22.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/master-2/logs
02:06:22.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/master-2/data/info.pb
02:06:22.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:22.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:06:22.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:06:22.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:06:22.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:06:22.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:22.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:22.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:22.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:06:22.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:06:22.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:06:22.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:06:22.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.228.252:39791
02:06:22.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.228.252
02:06:22.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:06:22.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.228.212:41215
02:06:22.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:22.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:22.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:22.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:06:22.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.228.254:43509,127.21.228.253:35519,127.21.228.252:39791
02:06:22.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:06:22.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41051/jwks.json with env {}
02:06:22.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:22.298586 22574 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:06:22.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:22.309707 22574 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.228.254" port: 43509 } has no permanent_uuid. Determining permanent_uuid...
02:06:22.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:22.332629 22574 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.228.253" port: 35519 } has no permanent_uuid. Determining permanent_uuid...
02:06:22.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:22.351372 22574 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.228.252" port: 39791 } has no permanent_uuid. Determining permanent_uuid...
02:06:22.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:22.353763 22523 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.21.228.252:39791: connect: Connection refused (error 111)
02:06:22.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:22.358094 22574 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.228.252:39791: Network error: Client connection negotiation failed: client connection to 127.21.228.252:39791: connect: Connection refused (error 111)
02:06:22.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:22.407637 22574 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.228.252" port: 39791 } attempt: 1
02:06:22.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:22.411469 22574 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.228.252:39791: Network error: Client connection negotiation failed: client connection to 127.21.228.252:39791: connect: Connection refused (error 111)
02:06:22.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:22.480268 22574 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.228.252" port: 39791 } attempt: 2
02:06:22.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:22.484241 22574 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.228.252:39791: Network error: Client connection negotiation failed: client connection to 127.21.228.252:39791: connect: Connection refused (error 111)
02:06:22.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:22.575685 22574 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.228.252" port: 39791 } attempt: 3
02:06:22.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:22.579869 22574 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.228.252:39791: Network error: Client connection negotiation failed: client connection to 127.21.228.252:39791: connect: Connection refused (error 111)
02:06:22.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:22.611352 22576 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:06:22.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:22.611897 22576 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:06:22.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:22.612457 22576 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:06:22.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:22.641961 22576 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:06:22.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:22.642246 22576 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:06:22.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:22.642428 22576 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:06:22.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:22.642760 22576 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:06:22.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:22.642962 22576 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:06:22.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:22.643112 22576 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41051/jwks.json
02:06:22.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:22.676324 22576 master_runner.cc:386] Master server non-default flags:
02:06:22.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:22.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.228.212:41215
02:06:22.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:22.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:22.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/master-2/data
02:06:22.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/master-2/wal
02:06:22.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:06:22.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.228.254:43509,127.21.228.253:35519,127.21.228.252:39791
02:06:22.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:22.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:22.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:06:22.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.228.252:39791
02:06:22.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:06:22.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:06:22.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:06:22.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41051/jwks.json
02:06:22.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:22.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:22.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/master-2/data/info.pb
02:06:22.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.228.252
02:06:22.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:06:22.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:06:22.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:22.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:06:22.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:06:22.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:22.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/master-2/logs
02:06:22.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:22.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:06:22.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:06:22.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:06:22.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:06:22.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
02:06:22.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:06:22.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 00:58:27 UTC on ccycloud.quasar-hahlai.root.comops.site
02:06:22.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4673502
02:06:22.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:06:22.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:22.677528 22576 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:06:22.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:22.679602 22576 file_cache.cc:493] Constructed file cache file cache with capacity 419430
02:06:22.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:22.699626 22587 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:22.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:22.723376 22574 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.228.252" port: 39791 } attempt: 4
02:06:22.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:22.727335 22574 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.228.252:39791: Network error: Client connection negotiation failed: client connection to 127.21.228.252:39791: connect: Connection refused (error 111)
02:06:22.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:22.853247 22499 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.228.253" port: 35519 } attempt: 7
02:06:22.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:22.867120 22499 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.228.252" port: 39791 } has no permanent_uuid. Determining permanent_uuid...
02:06:22.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:22.871877 22499 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.228.252:39791: Network error: Client connection negotiation failed: client connection to 127.21.228.252:39791: connect: Connection refused (error 111)
02:06:22.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:22.930481 22499 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.228.252" port: 39791 } attempt: 1
02:06:22.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:22.935554 22499 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.228.252:39791: Network error: Client connection negotiation failed: client connection to 127.21.228.252:39791: connect: Connection refused (error 111)
02:06:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:23.016928 22499 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.228.252" port: 39791 } attempt: 2
02:06:23.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:23.021520 22499 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.228.252:39791: Network error: Client connection negotiation failed: client connection to 127.21.228.252:39791: connect: Connection refused (error 111)
02:06:23.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:23.026892 22574 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.228.252" port: 39791 } attempt: 5
02:06:23.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:23.030673 22574 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.228.252:39791: Network error: Client connection negotiation failed: client connection to 127.21.228.252:39791: connect: Connection refused (error 111)
02:06:23.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:23.106897 22499 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.228.252" port: 39791 } attempt: 3
02:06:23.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:23.111670 22499 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.228.252:39791: Network error: Client connection negotiation failed: client connection to 127.21.228.252:39791: connect: Connection refused (error 111)
02:06:23.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:23.252063 22499 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.228.252" port: 39791 } attempt: 4
02:06:23.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:23.256418 22499 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.228.252:39791: Network error: Client connection negotiation failed: client connection to 127.21.228.252:39791: connect: Connection refused (error 111)
02:06:23.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:23.539809 22499 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.228.252" port: 39791 } attempt: 5
02:06:23.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:23.543282 22499 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.228.252:39791: Network error: Client connection negotiation failed: client connection to 127.21.228.252:39791: connect: Connection refused (error 111)
02:06:23.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:23.578249 22574 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.228.252" port: 39791 } attempt: 6
02:06:23.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:23.585464 22574 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.228.252:39791: Network error: Client connection negotiation failed: client connection to 127.21.228.252:39791: connect: Connection refused (error 111)
02:06:23.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:22.699664 22589 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:23.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:22.700801 22586 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:23.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:23.916652 22588 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
02:06:23.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:23.916786 22576 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:06:23.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:23.920799 22576 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:06:23.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:23.923763 22576 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:06:23.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:23.925192 22576 hybrid_clock.cc:648] HybridClock initialized: now 1790388383925164 us; error 59 us; skew 500 ppm
02:06:23.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:23.926095 22576 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:06:23.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:23.933728 22576 webserver.cc:460] Webserver started at http://127.21.228.252:33187/ using document root <none> and password file <none>
02:06:23.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:23.934916 22576 fs_manager.cc:359] Metadata directory not provided
02:06:23.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:23.935174 22576 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:06:23.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:23.935714 22576 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
02:06:23.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:23.942610 22576 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/master-2/data/instance:
02:06:23.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "074cd36f906d4df3baf14adbc89da3bf"
02:06:23.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 02:06:23 on dist-test-slave-nz2j"
02:06:23.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:23.944108 22576 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/master-2/wal/instance:
02:06:23.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "074cd36f906d4df3baf14adbc89da3bf"
02:06:23.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 02:06:23 on dist-test-slave-nz2j"
02:06:23.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:23.953495 22576 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.007s sys 0.000s
02:06:23.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:23.960085 22601 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:06:23.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:23.961220 22576 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
02:06:23.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:23.961506 22576 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/master-2/data,/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/master-2/wal
02:06:23.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "074cd36f906d4df3baf14adbc89da3bf"
02:06:23.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 02:06:23 on dist-test-slave-nz2j"
02:06:23.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:23.961862 22576 fs_report.cc:383] FS layout report
02:06:23.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:06:23.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/master-2/wal
02:06:23.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/master-2/wal
02:06:23.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/master-2/data/data
02:06:23.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:06:23.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:06:23.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:06:23.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:06:23.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:06:23.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:06:23.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:06:23.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:06:23.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:06:23.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:06:23.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:06:24.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.004572 22576 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
02:06:24.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.006572 22576 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:06:24.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.007292 22576 kserver.cc:163] Server-wide thread pool size limit: 3276
02:06:24.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.095854 22499 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.228.252" port: 39791 } attempt: 6
02:06:24.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:24.104133 22499 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.228.252:39791: Network error: Client connection negotiation failed: client connection to 127.21.228.252:39791: connect: Connection refused (error 111)
02:06:24.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.111704 22576 rpc_server.cc:307] RPC server started. Bound to: 127.21.228.252:39791
02:06:24.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.111843 22654 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.21.228.252:39791 every 8 connection(s)
02:06:24.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.114553 22576 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/master-2/data/info.pb
02:06:24.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.118263 22419 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQ73J9W/build/tsan/bin/kudu as pid 22576
02:06:24.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.118713 22419 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/master-2/wal/instance
02:06:24.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.120837 22655 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:06:24.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.139842 22655 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.228.254" port: 43509 } has no permanent_uuid. Determining permanent_uuid...
02:06:24.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.168006 22655 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.228.253" port: 35519 } has no permanent_uuid. Determining permanent_uuid...
02:06:24.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.180022 22655 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.228.252" port: 39791 } has no permanent_uuid. Determining permanent_uuid...
02:06:24.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.198024 22655 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 074cd36f906d4df3baf14adbc89da3bf: Bootstrap starting.
02:06:24.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.205127 22655 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 074cd36f906d4df3baf14adbc89da3bf: Neither blocks nor log segments found. Creating new log.
02:06:24.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.206595 22655 log.cc:824] T 00000000000000000000000000000000 P 074cd36f906d4df3baf14adbc89da3bf: Log is configured to *not* fsync() on all Append() calls
02:06:24.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.211771 22655 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 074cd36f906d4df3baf14adbc89da3bf: No bootstrap required, opened a new log
02:06:24.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.232668 22655 raft_consensus.cc:348] T 00000000000000000000000000000000 P 074cd36f906d4df3baf14adbc89da3bf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "723fe02fa81b44b08c4b52529d34d100" member_type: VOTER last_known_addr { host: "127.21.228.254" port: 43509 } } peers { permanent_uuid: "7891b700a4394fcfb8996911f72f37f0" member_type: VOTER last_known_addr { host: "127.21.228.253" port: 35519 } } peers { permanent_uuid: "074cd36f906d4df3baf14adbc89da3bf" member_type: VOTER last_known_addr { host: "127.21.228.252" port: 39791 } }
02:06:24.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.233280 22655 raft_consensus.cc:374] T 00000000000000000000000000000000 P 074cd36f906d4df3baf14adbc89da3bf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:06:24.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.233559 22655 raft_consensus.cc:729] T 00000000000000000000000000000000 P 074cd36f906d4df3baf14adbc89da3bf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 074cd36f906d4df3baf14adbc89da3bf, State: Initialized, Role: FOLLOWER
02:06:24.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.234339 22655 consensus_queue.cc:260] T 00000000000000000000000000000000 P 074cd36f906d4df3baf14adbc89da3bf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "723fe02fa81b44b08c4b52529d34d100" member_type: VOTER last_known_addr { host: "127.21.228.254" port: 43509 } } peers { permanent_uuid: "7891b700a4394fcfb8996911f72f37f0" member_type: VOTER last_known_addr { host: "127.21.228.253" port: 35519 } } peers { permanent_uuid: "074cd36f906d4df3baf14adbc89da3bf" member_type: VOTER last_known_addr { host: "127.21.228.252" port: 39791 } }
02:06:24.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.237363 22664 sys_catalog.cc:455] T 00000000000000000000000000000000 P 074cd36f906d4df3baf14adbc89da3bf [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "723fe02fa81b44b08c4b52529d34d100" member_type: VOTER last_known_addr { host: "127.21.228.254" port: 43509 } } peers { permanent_uuid: "7891b700a4394fcfb8996911f72f37f0" member_type: VOTER last_known_addr { host: "127.21.228.253" port: 35519 } } peers { permanent_uuid: "074cd36f906d4df3baf14adbc89da3bf" member_type: VOTER last_known_addr { host: "127.21.228.252" port: 39791 } } }
02:06:24.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.238080 22664 sys_catalog.cc:458] T 00000000000000000000000000000000 P 074cd36f906d4df3baf14adbc89da3bf [sys.catalog]: This master's current role is: FOLLOWER
02:06:24.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.238623 22655 sys_catalog.cc:564] T 00000000000000000000000000000000 P 074cd36f906d4df3baf14adbc89da3bf [sys.catalog]: configured and running, proceeding with master startup.
02:06:24.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:24.265111 22675 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 074cd36f906d4df3baf14adbc89da3bf: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:06:24.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:24.265407 22675 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:06:24.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:24.268275 22607 tablet.cc:2369] T 00000000000000000000000000000000 P 074cd36f906d4df3baf14adbc89da3bf: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:06:24.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.260497 22576 master_runner.cc:186] Error getting master registration for 127.21.228.254:43509: OK, instance_id { permanent_uuid: "723fe02fa81b44b08c4b52529d34d100" instance_seqno: 1790388380483933 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
02:06:24.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.294461 22576 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:06:24.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.356068 22664 raft_consensus.cc:482] T 00000000000000000000000000000000 P 074cd36f906d4df3baf14adbc89da3bf [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:06:24.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.356590 22664 raft_consensus.cc:504] T 00000000000000000000000000000000 P 074cd36f906d4df3baf14adbc89da3bf [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "723fe02fa81b44b08c4b52529d34d100" member_type: VOTER last_known_addr { host: "127.21.228.254" port: 43509 } } peers { permanent_uuid: "7891b700a4394fcfb8996911f72f37f0" member_type: VOTER last_known_addr { host: "127.21.228.253" port: 35519 } } peers { permanent_uuid: "074cd36f906d4df3baf14adbc89da3bf" member_type: VOTER last_known_addr { host: "127.21.228.252" port: 39791 } }
02:06:24.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.358814 22664 leader_election.cc:288] T 00000000000000000000000000000000 P 074cd36f906d4df3baf14adbc89da3bf [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 723fe02fa81b44b08c4b52529d34d100 (127.21.228.254:43509), 7891b700a4394fcfb8996911f72f37f0 (127.21.228.253:35519)
02:06:24.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.646014 22574 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.228.252" port: 39791 } attempt: 7
02:06:24.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.671248 22574 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7891b700a4394fcfb8996911f72f37f0: Bootstrap starting.
02:06:24.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.678974 22574 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 7891b700a4394fcfb8996911f72f37f0: Neither blocks nor log segments found. Creating new log.
02:06:24.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.683032 22574 log.cc:824] T 00000000000000000000000000000000 P 7891b700a4394fcfb8996911f72f37f0: Log is configured to *not* fsync() on all Append() calls
02:06:24.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.688832 22574 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7891b700a4394fcfb8996911f72f37f0: No bootstrap required, opened a new log
02:06:24.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.725339 22574 raft_consensus.cc:348] T 00000000000000000000000000000000 P 7891b700a4394fcfb8996911f72f37f0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "723fe02fa81b44b08c4b52529d34d100" member_type: VOTER last_known_addr { host: "127.21.228.254" port: 43509 } } peers { permanent_uuid: "7891b700a4394fcfb8996911f72f37f0" member_type: VOTER last_known_addr { host: "127.21.228.253" port: 35519 } } peers { permanent_uuid: "074cd36f906d4df3baf14adbc89da3bf" member_type: VOTER last_known_addr { host: "127.21.228.252" port: 39791 } }
02:06:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.726637 22574 raft_consensus.cc:374] T 00000000000000000000000000000000 P 7891b700a4394fcfb8996911f72f37f0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:06:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.727181 22574 raft_consensus.cc:729] T 00000000000000000000000000000000 P 7891b700a4394fcfb8996911f72f37f0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7891b700a4394fcfb8996911f72f37f0, State: Initialized, Role: FOLLOWER
02:06:24.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.728392 22574 consensus_queue.cc:260] T 00000000000000000000000000000000 P 7891b700a4394fcfb8996911f72f37f0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "723fe02fa81b44b08c4b52529d34d100" member_type: VOTER last_known_addr { host: "127.21.228.254" port: 43509 } } peers { permanent_uuid: "7891b700a4394fcfb8996911f72f37f0" member_type: VOTER last_known_addr { host: "127.21.228.253" port: 35519 } } peers { permanent_uuid: "074cd36f906d4df3baf14adbc89da3bf" member_type: VOTER last_known_addr { host: "127.21.228.252" port: 39791 } }
02:06:24.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.734102 22679 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7891b700a4394fcfb8996911f72f37f0 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "723fe02fa81b44b08c4b52529d34d100" member_type: VOTER last_known_addr { host: "127.21.228.254" port: 43509 } } peers { permanent_uuid: "7891b700a4394fcfb8996911f72f37f0" member_type: VOTER last_known_addr { host: "127.21.228.253" port: 35519 } } peers { permanent_uuid: "074cd36f906d4df3baf14adbc89da3bf" member_type: VOTER last_known_addr { host: "127.21.228.252" port: 39791 } } }
02:06:24.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.735000 22679 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7891b700a4394fcfb8996911f72f37f0 [sys.catalog]: This master's current role is: FOLLOWER
02:06:24.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.736063 22574 sys_catalog.cc:564] T 00000000000000000000000000000000 P 7891b700a4394fcfb8996911f72f37f0 [sys.catalog]: configured and running, proceeding with master startup.
02:06:24.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.736258 22549 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "074cd36f906d4df3baf14adbc89da3bf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7891b700a4394fcfb8996911f72f37f0" is_pre_election: true
02:06:24.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.737172 22549 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 7891b700a4394fcfb8996911f72f37f0 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 074cd36f906d4df3baf14adbc89da3bf in term 0.
02:06:24.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.738488 22604 leader_election.cc:302] T 00000000000000000000000000000000 P 074cd36f906d4df3baf14adbc89da3bf [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 074cd36f906d4df3baf14adbc89da3bf, 7891b700a4394fcfb8996911f72f37f0; no voters:
02:06:24.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.739564 22664 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 074cd36f906d4df3baf14adbc89da3bf [term 0 FOLLOWER]: Leader pre-election won for term 1
02:06:24.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.739879 22664 raft_consensus.cc:482] T 00000000000000000000000000000000 P 074cd36f906d4df3baf14adbc89da3bf [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
02:06:24.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.740446 22664 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 074cd36f906d4df3baf14adbc89da3bf [term 0 FOLLOWER]: Advancing to term 1
02:06:24.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.746678 22664 raft_consensus.cc:504] T 00000000000000000000000000000000 P 074cd36f906d4df3baf14adbc89da3bf [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "723fe02fa81b44b08c4b52529d34d100" member_type: VOTER last_known_addr { host: "127.21.228.254" port: 43509 } } peers { permanent_uuid: "7891b700a4394fcfb8996911f72f37f0" member_type: VOTER last_known_addr { host: "127.21.228.253" port: 35519 } } peers { permanent_uuid: "074cd36f906d4df3baf14adbc89da3bf" member_type: VOTER last_known_addr { host: "127.21.228.252" port: 39791 } }
02:06:24.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.751852 22664 leader_election.cc:288] T 00000000000000000000000000000000 P 074cd36f906d4df3baf14adbc89da3bf [CANDIDATE]: Term 1 election: Requested vote from peers 723fe02fa81b44b08c4b52529d34d100 (127.21.228.254:43509), 7891b700a4394fcfb8996911f72f37f0 (127.21.228.253:35519)
02:06:24.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.752852 22549 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "074cd36f906d4df3baf14adbc89da3bf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7891b700a4394fcfb8996911f72f37f0"
02:06:24.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.754067 22549 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 7891b700a4394fcfb8996911f72f37f0 [term 0 FOLLOWER]: Advancing to term 1
02:06:24.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.764530 22549 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 7891b700a4394fcfb8996911f72f37f0 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 074cd36f906d4df3baf14adbc89da3bf in term 1.
02:06:24.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.771946 22604 leader_election.cc:302] T 00000000000000000000000000000000 P 074cd36f906d4df3baf14adbc89da3bf [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 074cd36f906d4df3baf14adbc89da3bf, 7891b700a4394fcfb8996911f72f37f0; no voters:
02:06:24.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.774049 22664 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 074cd36f906d4df3baf14adbc89da3bf [term 1 FOLLOWER]: Leader election won for term 1
02:06:24.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.780359 22664 raft_consensus.cc:686] T 00000000000000000000000000000000 P 074cd36f906d4df3baf14adbc89da3bf [term 1 LEADER]: Becoming Leader. State: Replica: 074cd36f906d4df3baf14adbc89da3bf, State: Running, Role: LEADER
02:06:24.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.781359 22690 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
02:06:24.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.781454 22664 consensus_queue.cc:237] T 00000000000000000000000000000000 P 074cd36f906d4df3baf14adbc89da3bf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "723fe02fa81b44b08c4b52529d34d100" member_type: VOTER last_known_addr { host: "127.21.228.254" port: 43509 } } peers { permanent_uuid: "7891b700a4394fcfb8996911f72f37f0" member_type: VOTER last_known_addr { host: "127.21.228.253" port: 35519 } } peers { permanent_uuid: "074cd36f906d4df3baf14adbc89da3bf" member_type: VOTER last_known_addr { host: "127.21.228.252" port: 39791 } }
02:06:24.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.794456 22691 sys_catalog.cc:455] T 00000000000000000000000000000000 P 074cd36f906d4df3baf14adbc89da3bf [sys.catalog]: SysCatalogTable state changed. Reason: New leader 074cd36f906d4df3baf14adbc89da3bf. Latest consensus state: current_term: 1 leader_uuid: "074cd36f906d4df3baf14adbc89da3bf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "723fe02fa81b44b08c4b52529d34d100" member_type: VOTER last_known_addr { host: "127.21.228.254" port: 43509 } } peers { permanent_uuid: "7891b700a4394fcfb8996911f72f37f0" member_type: VOTER last_known_addr { host: "127.21.228.253" port: 35519 } } peers { permanent_uuid: "074cd36f906d4df3baf14adbc89da3bf" member_type: VOTER last_known_addr { host: "127.21.228.252" port: 39791 } } }
02:06:24.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.795317 22691 sys_catalog.cc:458] T 00000000000000000000000000000000 P 074cd36f906d4df3baf14adbc89da3bf [sys.catalog]: This master's current role is: LEADER
02:06:24.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.798743 22693 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
02:06:24.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.819178 22693 catalog_manager.cc:1484] Initializing Kudu cluster ID...
02:06:24.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.784487 22501 master_runner.cc:186] Error getting master registration for 127.21.228.254:43509: OK, instance_id { permanent_uuid: "723fe02fa81b44b08c4b52529d34d100" instance_seqno: 1790388380483933 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
02:06:24.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.821879 22501 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:06:24.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.837850 22549 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 7891b700a4394fcfb8996911f72f37f0 [term 1 FOLLOWER]: Refusing update from remote peer 074cd36f906d4df3baf14adbc89da3bf: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
02:06:24.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.839840 22664 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 074cd36f906d4df3baf14adbc89da3bf [LEADER]: Connected to new peer: Peer: permanent_uuid: "7891b700a4394fcfb8996911f72f37f0" member_type: VOTER last_known_addr { host: "127.21.228.253" port: 35519 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:06:24.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.879233 22679 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7891b700a4394fcfb8996911f72f37f0 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 074cd36f906d4df3baf14adbc89da3bf. Latest consensus state: current_term: 1 leader_uuid: "074cd36f906d4df3baf14adbc89da3bf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "723fe02fa81b44b08c4b52529d34d100" member_type: VOTER last_known_addr { host: "127.21.228.254" port: 43509 } } peers { permanent_uuid: "7891b700a4394fcfb8996911f72f37f0" member_type: VOTER last_known_addr { host: "127.21.228.253" port: 35519 } } peers { permanent_uuid: "074cd36f906d4df3baf14adbc89da3bf" member_type: VOTER last_known_addr { host: "127.21.228.252" port: 39791 } } }
02:06:24.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.880875 22694 mvcc.cc:205] Tried to move back new op lower bound from 7333430824276819968 to 7333430824075497472. Current Snapshot: MvccSnapshot[applied={T|T < 7333430824276819968}]
02:06:24.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.881935 22679 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7891b700a4394fcfb8996911f72f37f0 [sys.catalog]: This master's current role is: FOLLOWER
02:06:24.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.883340 22691 sys_catalog.cc:455] T 00000000000000000000000000000000 P 074cd36f906d4df3baf14adbc89da3bf [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "074cd36f906d4df3baf14adbc89da3bf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "723fe02fa81b44b08c4b52529d34d100" member_type: VOTER last_known_addr { host: "127.21.228.254" port: 43509 } } peers { permanent_uuid: "7891b700a4394fcfb8996911f72f37f0" member_type: VOTER last_known_addr { host: "127.21.228.253" port: 35519 } } peers { permanent_uuid: "074cd36f906d4df3baf14adbc89da3bf" member_type: VOTER last_known_addr { host: "127.21.228.252" port: 39791 } } }
02:06:24.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.884043 22691 sys_catalog.cc:458] T 00000000000000000000000000000000 P 074cd36f906d4df3baf14adbc89da3bf [sys.catalog]: This master's current role is: LEADER
02:06:24.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.889093 22693 catalog_manager.cc:1347] Generated new cluster ID: 9d71979bcd01441fbdd100d85356e61f
02:06:24.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.889397 22693 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
02:06:24.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.896358 22679 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7891b700a4394fcfb8996911f72f37f0 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "074cd36f906d4df3baf14adbc89da3bf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "723fe02fa81b44b08c4b52529d34d100" member_type: VOTER last_known_addr { host: "127.21.228.254" port: 43509 } } peers { permanent_uuid: "7891b700a4394fcfb8996911f72f37f0" member_type: VOTER last_known_addr { host: "127.21.228.253" port: 35519 } } peers { permanent_uuid: "074cd36f906d4df3baf14adbc89da3bf" member_type: VOTER last_known_addr { host: "127.21.228.252" port: 39791 } } }
02:06:24.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.897939 22679 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7891b700a4394fcfb8996911f72f37f0 [sys.catalog]: This master's current role is: FOLLOWER
02:06:24.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.919158 22693 catalog_manager.cc:1370] Generated new certificate authority record
02:06:24.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.922006 22693 catalog_manager.cc:1504] Loading token signing keys...
02:06:24.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.947692 22693 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 074cd36f906d4df3baf14adbc89da3bf: Generated new TSK 0
02:06:24.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.949002 22693 catalog_manager.cc:1514] Initializing in-progress tserver states...
02:06:24.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:24.987051 22419 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQ73J9W/build/tsan/bin/kudu
02:06:24.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQ73J9W/build/tsan/bin/kudu
02:06:24.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/ts-0/wal
02:06:24.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/ts-0/data
02:06:24.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:06:24.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:06:24.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:06:24.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:24.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:24.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:24.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/ts-0/logs
02:06:24.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/ts-0/data/info.pb
02:06:24.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:24.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:06:24.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:06:24.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:06:24.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:06:24.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:24.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:24.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:24.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:06:24.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:06:24.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.228.193:0
02:06:24.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.228.193
02:06:24.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.228.193
02:06:24.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:06:24.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.228.254:43509,127.21.228.253:35519,127.21.228.252:39791
02:06:24.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.228.212:41215
02:06:24.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:24.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:24.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
02:06:25.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:25.137820 22499 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.228.252" port: 39791 } attempt: 7
02:06:25.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:25.175098 22499 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 723fe02fa81b44b08c4b52529d34d100: Bootstrap starting.
02:06:25.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:25.182550 22499 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 723fe02fa81b44b08c4b52529d34d100: Neither blocks nor log segments found. Creating new log.
02:06:25.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:25.184728 22499 log.cc:824] T 00000000000000000000000000000000 P 723fe02fa81b44b08c4b52529d34d100: Log is configured to *not* fsync() on all Append() calls
02:06:25.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:25.191742 22499 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 723fe02fa81b44b08c4b52529d34d100: No bootstrap required, opened a new log
02:06:25.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:25.231858 22499 raft_consensus.cc:348] T 00000000000000000000000000000000 P 723fe02fa81b44b08c4b52529d34d100 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "723fe02fa81b44b08c4b52529d34d100" member_type: VOTER last_known_addr { host: "127.21.228.254" port: 43509 } } peers { permanent_uuid: "7891b700a4394fcfb8996911f72f37f0" member_type: VOTER last_known_addr { host: "127.21.228.253" port: 35519 } } peers { permanent_uuid: "074cd36f906d4df3baf14adbc89da3bf" member_type: VOTER last_known_addr { host: "127.21.228.252" port: 39791 } }
02:06:25.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:25.233042 22499 raft_consensus.cc:374] T 00000000000000000000000000000000 P 723fe02fa81b44b08c4b52529d34d100 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:06:25.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:25.233336 22499 raft_consensus.cc:729] T 00000000000000000000000000000000 P 723fe02fa81b44b08c4b52529d34d100 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 723fe02fa81b44b08c4b52529d34d100, State: Initialized, Role: FOLLOWER
02:06:25.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:25.234408 22499 consensus_queue.cc:260] T 00000000000000000000000000000000 P 723fe02fa81b44b08c4b52529d34d100 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "723fe02fa81b44b08c4b52529d34d100" member_type: VOTER last_known_addr { host: "127.21.228.254" port: 43509 } } peers { permanent_uuid: "7891b700a4394fcfb8996911f72f37f0" member_type: VOTER last_known_addr { host: "127.21.228.253" port: 35519 } } peers { permanent_uuid: "074cd36f906d4df3baf14adbc89da3bf" member_type: VOTER last_known_addr { host: "127.21.228.252" port: 39791 } }
02:06:25.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:25.240123 22499 sys_catalog.cc:564] T 00000000000000000000000000000000 P 723fe02fa81b44b08c4b52529d34d100 [sys.catalog]: configured and running, proceeding with master startup.
02:06:25.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:25.240360 22473 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "074cd36f906d4df3baf14adbc89da3bf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "723fe02fa81b44b08c4b52529d34d100"
02:06:25.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:25.241184 22473 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 723fe02fa81b44b08c4b52529d34d100 [term 0 FOLLOWER]: Advancing to term 1
02:06:25.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:25.242589 22474 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "074cd36f906d4df3baf14adbc89da3bf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "723fe02fa81b44b08c4b52529d34d100" is_pre_election: true
02:06:25.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:25.249486 22473 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 723fe02fa81b44b08c4b52529d34d100 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 074cd36f906d4df3baf14adbc89da3bf in term 1.
02:06:25.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:25.252957 22472 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 723fe02fa81b44b08c4b52529d34d100 [term 1 FOLLOWER]: Refusing update from remote peer 074cd36f906d4df3baf14adbc89da3bf: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
02:06:25.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:25.252908 22702 sys_catalog.cc:455] T 00000000000000000000000000000000 P 723fe02fa81b44b08c4b52529d34d100 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "723fe02fa81b44b08c4b52529d34d100" member_type: VOTER last_known_addr { host: "127.21.228.254" port: 43509 } } peers { permanent_uuid: "7891b700a4394fcfb8996911f72f37f0" member_type: VOTER last_known_addr { host: "127.21.228.253" port: 35519 } } peers { permanent_uuid: "074cd36f906d4df3baf14adbc89da3bf" member_type: VOTER last_known_addr { host: "127.21.228.252" port: 39791 } } }
02:06:25.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:25.253741 22702 sys_catalog.cc:458] T 00000000000000000000000000000000 P 723fe02fa81b44b08c4b52529d34d100 [sys.catalog]: This master's current role is: FOLLOWER
02:06:25.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:25.262815 22664 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 074cd36f906d4df3baf14adbc89da3bf [LEADER]: Connected to new peer: Peer: permanent_uuid: "723fe02fa81b44b08c4b52529d34d100" member_type: VOTER last_known_addr { host: "127.21.228.254" port: 43509 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:06:25.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:25.318986 22702 sys_catalog.cc:455] T 00000000000000000000000000000000 P 723fe02fa81b44b08c4b52529d34d100 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 074cd36f906d4df3baf14adbc89da3bf. Latest consensus state: current_term: 1 leader_uuid: "074cd36f906d4df3baf14adbc89da3bf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "723fe02fa81b44b08c4b52529d34d100" member_type: VOTER last_known_addr { host: "127.21.228.254" port: 43509 } } peers { permanent_uuid: "7891b700a4394fcfb8996911f72f37f0" member_type: VOTER last_known_addr { host: "127.21.228.253" port: 35519 } } peers { permanent_uuid: "074cd36f906d4df3baf14adbc89da3bf" member_type: VOTER last_known_addr { host: "127.21.228.252" port: 39791 } } }
02:06:25.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:25.319684 22702 sys_catalog.cc:458] T 00000000000000000000000000000000 P 723fe02fa81b44b08c4b52529d34d100 [sys.catalog]: This master's current role is: FOLLOWER
02:06:25.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:25.320660 22709 sys_catalog.cc:455] T 00000000000000000000000000000000 P 723fe02fa81b44b08c4b52529d34d100 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "074cd36f906d4df3baf14adbc89da3bf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "723fe02fa81b44b08c4b52529d34d100" member_type: VOTER last_known_addr { host: "127.21.228.254" port: 43509 } } peers { permanent_uuid: "7891b700a4394fcfb8996911f72f37f0" member_type: VOTER last_known_addr { host: "127.21.228.253" port: 35519 } } peers { permanent_uuid: "074cd36f906d4df3baf14adbc89da3bf" member_type: VOTER last_known_addr { host: "127.21.228.252" port: 39791 } } }
02:06:25.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:25.321576 22709 sys_catalog.cc:458] T 00000000000000000000000000000000 P 723fe02fa81b44b08c4b52529d34d100 [sys.catalog]: This master's current role is: FOLLOWER
02:06:25.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:25.423714 22719 catalog_manager.cc:1259] Loaded cluster ID: 9d71979bcd01441fbdd100d85356e61f
02:06:25.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:25.423941 22719 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 723fe02fa81b44b08c4b52529d34d100: loading cluster ID for follower catalog manager: success
02:06:25.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:25.430032 22719 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 723fe02fa81b44b08c4b52529d34d100: acquiring CA information for follower catalog manager: success
02:06:25.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:25.435392 22719 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 723fe02fa81b44b08c4b52529d34d100: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:06:25.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:25.525358 22699 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:06:25.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:25.526330 22699 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:06:25.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:25.527259 22699 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:06:25.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:25.581670 22699 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:06:25.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:25.583222 22699 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.21.228.193
02:06:25.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:25.641776 22699 tablet_server_runner.cc:78] Tablet server non-default flags:
02:06:25.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:25.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.228.212:41215
02:06:25.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:25.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:25.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/ts-0/data
02:06:25.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/ts-0/wal
02:06:25.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:25.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:25.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.228.193:0
02:06:25.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:06:25.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:25.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:25.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/ts-0/data/info.pb
02:06:25.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.228.193
02:06:25.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:06:25.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.228.254:43509,127.21.228.253:35519,127.21.228.252:39791
02:06:25.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:06:25.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:25.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:06:25.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:06:25.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:25.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.228.193
02:06:25.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/ts-0/logs
02:06:25.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:25.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:06:25.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:06:25.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:06:25.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:06:25.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
02:06:25.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:06:25.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 00:58:27 UTC on ccycloud.quasar-hahlai.root.comops.site
02:06:25.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4673502
02:06:25.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:06:25.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:25.643249 22699 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:06:25.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:25.645223 22699 file_cache.cc:493] Constructed file cache file cache with capacity 419430
02:06:25.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:25.672544 22726 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:25.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:25.814177 22690 catalog_manager.cc:1259] Loaded cluster ID: 9d71979bcd01441fbdd100d85356e61f
02:06:25.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:25.824340 22690 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 7891b700a4394fcfb8996911f72f37f0: loading cluster ID for follower catalog manager: success
02:06:25.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:25.844902 22690 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 7891b700a4394fcfb8996911f72f37f0: acquiring CA information for follower catalog manager: success
02:06:25.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:25.880470 22690 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 7891b700a4394fcfb8996911f72f37f0: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:06:25.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:25.904178 22664 sys_catalog.cc:455] T 00000000000000000000000000000000 P 074cd36f906d4df3baf14adbc89da3bf [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "074cd36f906d4df3baf14adbc89da3bf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "723fe02fa81b44b08c4b52529d34d100" member_type: VOTER last_known_addr { host: "127.21.228.254" port: 43509 } } peers { permanent_uuid: "7891b700a4394fcfb8996911f72f37f0" member_type: VOTER last_known_addr { host: "127.21.228.253" port: 35519 } } peers { permanent_uuid: "074cd36f906d4df3baf14adbc89da3bf" member_type: VOTER last_known_addr { host: "127.21.228.252" port: 39791 } } }
02:06:25.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:25.905839 22664 sys_catalog.cc:458] T 00000000000000000000000000000000 P 074cd36f906d4df3baf14adbc89da3bf [sys.catalog]: This master's current role is: LEADER
02:06:28.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:27.073623 22725 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 22699
02:06:28.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:28.730823 22725 kernel_stack_watchdog.cc:198] Thread 22699 stuck at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:637 for 402ms:
02:06:28.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
02:06:28.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
02:06:28.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
02:06:28.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
02:06:28.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:28.731391 22699 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 3.062s user 0.938s sys 1.694s
02:06:28.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:28.731731 22699 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 3.063s user 0.938s sys 1.694s
02:06:28.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:25.672945 22727 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:28.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:28.734432 22729 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:28.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:28.738454 22728 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 3062 milliseconds
02:06:28.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:28.738490 22699 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:06:28.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:28.742071 22699 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:06:28.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:28.744735 22699 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:06:28.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:28.746188 22699 hybrid_clock.cc:648] HybridClock initialized: now 1790388388746034 us; error 158 us; skew 500 ppm
02:06:28.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:28.747241 22699 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:06:28.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:28.762516 22699 webserver.cc:460] Webserver started at http://127.21.228.193:37675/ using document root <none> and password file <none>
02:06:28.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:28.763731 22699 fs_manager.cc:359] Metadata directory not provided
02:06:28.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:28.763999 22699 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:06:28.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:28.764571 22699 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
02:06:28.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:28.771461 22699 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/ts-0/data/instance:
02:06:28.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "095e0137633e4b67b87ac88714455a8e"
02:06:28.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 02:06:28 on dist-test-slave-nz2j"
02:06:28.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:28.772941 22699 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/ts-0/wal/instance:
02:06:28.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "095e0137633e4b67b87ac88714455a8e"
02:06:28.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 02:06:28 on dist-test-slave-nz2j"
02:06:28.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:28.782531 22699 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.004s sys 0.004s
02:06:28.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:28.790282 22741 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:06:28.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:28.791550 22699 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.006s sys 0.000s
02:06:28.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:28.791916 22699 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/ts-0/data,/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/ts-0/wal
02:06:28.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "095e0137633e4b67b87ac88714455a8e"
02:06:28.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 02:06:28 on dist-test-slave-nz2j"
02:06:28.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:28.792371 22699 fs_report.cc:383] FS layout report
02:06:28.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:06:28.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/ts-0/wal
02:06:28.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/ts-0/wal
02:06:28.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/ts-0/data/data
02:06:28.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:06:28.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:06:28.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:06:28.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:06:28.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:06:28.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:06:28.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:06:28.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:06:28.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:06:28.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:06:28.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:06:28.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:28.849395 22699 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
02:06:28.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:28.851303 22699 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:06:28.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:28.851917 22699 kserver.cc:163] Server-wide thread pool size limit: 3276
02:06:28.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:28.855279 22699 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
02:06:28.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:28.861248 22699 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:06:28.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:28.861514 22699 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
02:06:28.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:28.861800 22699 ts_tablet_manager.cc:610] Registered 0 tablets
02:06:28.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:28.862001 22699 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:06:29.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:29.083323 22699 rpc_server.cc:307] RPC server started. Bound to: 127.21.228.193:41319
02:06:29.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:29.083447 22854 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.21.228.193:41319 every 8 connection(s)
02:06:29.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:29.087251 22699 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/ts-0/data/info.pb
02:06:29.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:29.088112 22419 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQ73J9W/build/tsan/bin/kudu as pid 22699
02:06:29.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:29.088771 22419 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/ts-0/wal/instance
02:06:29.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:29.099073 22419 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQ73J9W/build/tsan/bin/kudu
02:06:29.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQ73J9W/build/tsan/bin/kudu
02:06:29.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/ts-1/wal
02:06:29.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/ts-1/data
02:06:29.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:06:29.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:06:29.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:06:29.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:29.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:29.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:29.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/ts-1/logs
02:06:29.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/ts-1/data/info.pb
02:06:29.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:29.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:06:29.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:06:29.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:06:29.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:06:29.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:29.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:29.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:29.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:06:29.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:06:29.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.228.194:0
02:06:29.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.228.194
02:06:29.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.228.194
02:06:29.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:06:29.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.228.254:43509,127.21.228.253:35519,127.21.228.252:39791
02:06:29.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.228.212:41215
02:06:29.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:29.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:29.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
02:06:29.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:29.163260 22855 heartbeater.cc:347] Connected to a master server at 127.21.228.252:39791
02:06:29.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:29.163746 22855 heartbeater.cc:464] Registering TS with master...
02:06:29.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:29.164979 22855 heartbeater.cc:511] Master 127.21.228.252:39791 requested a full tablet report, sending...
02:06:29.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:29.168357 22618 ts_manager.cc:194] Registered new tserver with Master: 095e0137633e4b67b87ac88714455a8e (127.21.228.193:41319)
02:06:29.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:29.171648 22618 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.21.228.193:48525
02:06:29.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:29.181831 22858 heartbeater.cc:347] Connected to a master server at 127.21.228.253:35519
02:06:29.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:29.182235 22858 heartbeater.cc:464] Registering TS with master...
02:06:29.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:29.183027 22858 heartbeater.cc:511] Master 127.21.228.253:35519 requested a full tablet report, sending...
02:06:29.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:29.186230 22539 ts_manager.cc:194] Registered new tserver with Master: 095e0137633e4b67b87ac88714455a8e (127.21.228.193:41319)
02:06:29.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:29.197953 22860 heartbeater.cc:347] Connected to a master server at 127.21.228.254:43509
02:06:29.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:29.198279 22860 heartbeater.cc:464] Registering TS with master...
02:06:29.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:29.199004 22860 heartbeater.cc:511] Master 127.21.228.254:43509 requested a full tablet report, sending...
02:06:29.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:29.201846 22464 ts_manager.cc:194] Registered new tserver with Master: 095e0137633e4b67b87ac88714455a8e (127.21.228.193:41319)
02:06:29.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:29.578714 22859 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:06:29.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:29.579352 22859 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:06:29.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:29.579844 22859 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:06:29.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:29.614658 22859 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:06:29.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:29.615815 22859 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.21.228.194
02:06:29.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:29.655604 22859 tablet_server_runner.cc:78] Tablet server non-default flags:
02:06:29.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:29.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.228.212:41215
02:06:29.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:29.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:29.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/ts-1/data
02:06:29.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/ts-1/wal
02:06:29.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:29.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:29.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.228.194:0
02:06:29.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:06:29.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:29.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:29.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/ts-1/data/info.pb
02:06:29.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.228.194
02:06:29.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:06:29.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.228.254:43509,127.21.228.253:35519,127.21.228.252:39791
02:06:29.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:06:29.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:29.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:06:29.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:06:29.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:29.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.228.194
02:06:29.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/ts-1/logs
02:06:29.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:29.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:06:29.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:06:29.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:06:29.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:06:29.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
02:06:29.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:06:29.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 00:58:27 UTC on ccycloud.quasar-hahlai.root.comops.site
02:06:29.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4673502
02:06:29.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:06:29.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:29.657285 22859 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:06:29.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:29.660122 22859 file_cache.cc:493] Constructed file cache file cache with capacity 419430
02:06:29.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:29.695703 22871 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:30.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:30.178995 22855 heartbeater.cc:503] Master 127.21.228.252:39791 was elected leader, sending a full tablet report...
02:06:31.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:29.695711 22873 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:31.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:29.695863 22870 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:31.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:31.031032 22872 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1334 milliseconds
02:06:31.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:31.031143 22859 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:06:31.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:31.032537 22859 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:06:31.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:31.034869 22859 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:06:31.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:31.036293 22859 hybrid_clock.cc:648] HybridClock initialized: now 1790388391036233 us; error 75 us; skew 500 ppm
02:06:31.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:31.037110 22859 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:06:31.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:31.043612 22859 webserver.cc:460] Webserver started at http://127.21.228.194:46833/ using document root <none> and password file <none>
02:06:31.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:31.044559 22859 fs_manager.cc:359] Metadata directory not provided
02:06:31.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:31.044751 22859 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:06:31.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:31.045116 22859 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
02:06:31.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:31.050072 22859 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/ts-1/data/instance:
02:06:31.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c9ad26a70c2a4fe1827b11615f1d01c1"
02:06:31.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 02:06:31 on dist-test-slave-nz2j"
02:06:31.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:31.051129 22859 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/ts-1/wal/instance:
02:06:31.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c9ad26a70c2a4fe1827b11615f1d01c1"
02:06:31.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 02:06:31 on dist-test-slave-nz2j"
02:06:31.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:31.058328 22859 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.001s sys 0.008s
02:06:31.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:31.063663 22881 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:06:31.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:31.064975 22859 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
02:06:31.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:31.065258 22859 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/ts-1/data,/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/ts-1/wal
02:06:31.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c9ad26a70c2a4fe1827b11615f1d01c1"
02:06:31.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 02:06:31 on dist-test-slave-nz2j"
02:06:31.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:31.065589 22859 fs_report.cc:383] FS layout report
02:06:31.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:06:31.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/ts-1/wal
02:06:31.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/ts-1/wal
02:06:31.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/ts-1/data/data
02:06:31.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:06:31.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:06:31.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:06:31.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:06:31.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:06:31.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:06:31.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:06:31.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:06:31.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:06:31.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:06:31.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:06:31.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:31.102520 22859 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
02:06:31.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:31.104027 22859 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:06:31.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:31.104496 22859 kserver.cc:163] Server-wide thread pool size limit: 3276
02:06:31.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:31.106861 22859 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
02:06:31.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:31.111680 22859 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:06:31.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:31.111866 22859 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
02:06:31.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:31.112053 22859 ts_tablet_manager.cc:610] Registered 0 tablets
02:06:31.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:31.112182 22859 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:06:31.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:31.332436 22859 rpc_server.cc:307] RPC server started. Bound to: 127.21.228.194:33523
02:06:31.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:31.332710 22993 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.21.228.194:33523 every 8 connection(s)
02:06:31.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:31.335296 22859 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/ts-1/data/info.pb
02:06:31.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:31.345515 22419 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQ73J9W/build/tsan/bin/kudu as pid 22859
02:06:31.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:31.346083 22419 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/ts-1/wal/instance
02:06:31.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:31.381423 22419 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQ73J9W/build/tsan/bin/kudu
02:06:31.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQ73J9W/build/tsan/bin/kudu
02:06:31.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/ts-2/wal
02:06:31.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/ts-2/data
02:06:31.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:06:31.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:06:31.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:06:31.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:31.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:31.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:31.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/ts-2/logs
02:06:31.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/ts-2/data/info.pb
02:06:31.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:31.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:06:31.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:06:31.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:06:31.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:06:31.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:31.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:31.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:31.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:06:31.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:06:31.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.228.195:0
02:06:31.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.228.195
02:06:31.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.228.195
02:06:31.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:06:31.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.228.254:43509,127.21.228.253:35519,127.21.228.252:39791
02:06:31.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.228.212:41215
02:06:31.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:31.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:31.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
02:06:31.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:31.432371 22995 heartbeater.cc:347] Connected to a master server at 127.21.228.253:35519
02:06:31.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:31.433005 22995 heartbeater.cc:464] Registering TS with master...
02:06:31.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:31.434680 22995 heartbeater.cc:511] Master 127.21.228.253:35519 requested a full tablet report, sending...
02:06:31.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:31.440753 22539 ts_manager.cc:194] Registered new tserver with Master: c9ad26a70c2a4fe1827b11615f1d01c1 (127.21.228.194:33523)
02:06:31.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:31.444501 22994 heartbeater.cc:347] Connected to a master server at 127.21.228.252:39791
02:06:31.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:31.444818 22994 heartbeater.cc:464] Registering TS with master...
02:06:31.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:31.445636 22994 heartbeater.cc:511] Master 127.21.228.252:39791 requested a full tablet report, sending...
02:06:31.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:31.447818 22618 ts_manager.cc:194] Registered new tserver with Master: c9ad26a70c2a4fe1827b11615f1d01c1 (127.21.228.194:33523)
02:06:31.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:31.448137 23000 heartbeater.cc:347] Connected to a master server at 127.21.228.254:43509
02:06:31.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:31.448506 23000 heartbeater.cc:464] Registering TS with master...
02:06:31.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:31.449203 23000 heartbeater.cc:511] Master 127.21.228.254:43509 requested a full tablet report, sending...
02:06:31.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:31.449855 22618 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.21.228.194:59151
02:06:31.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:31.451526 22464 ts_manager.cc:194] Registered new tserver with Master: c9ad26a70c2a4fe1827b11615f1d01c1 (127.21.228.194:33523)
02:06:31.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:31.744009 23004 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:06:31.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:31.744545 23004 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:06:31.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:31.744976 23004 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:06:31.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:31.777542 23004 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:06:31.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:31.778332 23004 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.21.228.195
02:06:31.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:31.813948 23004 tablet_server_runner.cc:78] Tablet server non-default flags:
02:06:31.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:31.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.228.212:41215
02:06:31.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:31.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:31.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/ts-2/data
02:06:31.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/ts-2/wal
02:06:31.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:31.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:31.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.228.195:0
02:06:31.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:06:31.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:31.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:31.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/ts-2/data/info.pb
02:06:31.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.228.195
02:06:31.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:06:31.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.228.254:43509,127.21.228.253:35519,127.21.228.252:39791
02:06:31.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:06:31.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:31.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:06:31.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:06:31.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:31.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.228.195
02:06:31.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/ts-2/logs
02:06:31.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:31.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:06:31.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:06:31.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:06:31.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:06:31.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
02:06:31.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:06:31.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 00:58:27 UTC on ccycloud.quasar-hahlai.root.comops.site
02:06:31.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4673502
02:06:31.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:06:31.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:31.815132 23004 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:06:31.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:31.816742 23004 file_cache.cc:493] Constructed file cache file cache with capacity 419430
02:06:31.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:31.837052 23010 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:32.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:32.454223 22994 heartbeater.cc:503] Master 127.21.228.252:39791 was elected leader, sending a full tablet report...
02:06:33.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:31.837131 23011 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:33.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:31.837436 23013 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:33.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:33.164263 23012 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1329 milliseconds
02:06:33.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:33.164390 23004 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:06:33.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:33.169077 23004 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:06:33.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:33.171466 23004 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:06:33.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:33.172921 23004 hybrid_clock.cc:648] HybridClock initialized: now 1790388393172877 us; error 44 us; skew 500 ppm
02:06:33.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:33.173889 23004 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:06:33.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:33.181334 23004 webserver.cc:460] Webserver started at http://127.21.228.195:37521/ using document root <none> and password file <none>
02:06:33.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:33.182345 23004 fs_manager.cc:359] Metadata directory not provided
02:06:33.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:33.182616 23004 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:06:33.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:33.183171 23004 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
02:06:33.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:33.188686 23004 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/ts-2/data/instance:
02:06:33.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f6db82e7e7554c0290a1c6a9e4b692cf"
02:06:33.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 02:06:33 on dist-test-slave-nz2j"
02:06:33.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:33.190085 23004 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/ts-2/wal/instance:
02:06:33.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f6db82e7e7554c0290a1c6a9e4b692cf"
02:06:33.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 02:06:33 on dist-test-slave-nz2j"
02:06:33.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:33.197571 23004 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.008s sys 0.000s
02:06:33.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:33.202858 23021 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:06:33.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:33.203903 23004 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
02:06:33.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:33.204339 23004 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/ts-2/data,/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/ts-2/wal
02:06:33.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f6db82e7e7554c0290a1c6a9e4b692cf"
02:06:33.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 02:06:33 on dist-test-slave-nz2j"
02:06:33.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:33.204767 23004 fs_report.cc:383] FS layout report
02:06:33.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:06:33.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/ts-2/wal
02:06:33.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/ts-2/wal
02:06:33.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/ts-2/data/data
02:06:33.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:06:33.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:06:33.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:06:33.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:06:33.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:06:33.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:06:33.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:06:33.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:06:33.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:06:33.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:06:33.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:06:33.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:33.247265 23004 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
02:06:33.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:33.248792 23004 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:06:33.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:33.249188 23004 kserver.cc:163] Server-wide thread pool size limit: 3276
02:06:33.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:33.251591 23004 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
02:06:33.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:33.255834 23004 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:06:33.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:33.256059 23004 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
02:06:33.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:33.256400 23004 ts_tablet_manager.cc:610] Registered 0 tablets
02:06:33.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:33.256600 23004 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:06:33.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:33.419435 23004 rpc_server.cc:307] RPC server started. Bound to: 127.21.228.195:35067
02:06:33.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:33.419570 23133 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.21.228.195:35067 every 8 connection(s)
02:06:33.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:33.422392 23004 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/ts-2/data/info.pb
02:06:33.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:33.428647 22419 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQ73J9W/build/tsan/bin/kudu as pid 23004
02:06:33.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:33.429201 22419 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1009842836068184325/ts-2/wal/instance
02:06:33.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:33.462239 23135 heartbeater.cc:347] Connected to a master server at 127.21.228.253:35519
02:06:33.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:33.462746 23135 heartbeater.cc:464] Registering TS with master...
02:06:33.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:33.464391 23135 heartbeater.cc:511] Master 127.21.228.253:35519 requested a full tablet report, sending...
02:06:33.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:33.466116 23134 heartbeater.cc:347] Connected to a master server at 127.21.228.252:39791
02:06:33.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:33.466477 23134 heartbeater.cc:464] Registering TS with master...
02:06:33.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:33.467198 23134 heartbeater.cc:511] Master 127.21.228.252:39791 requested a full tablet report, sending...
02:06:33.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:33.467268 22539 ts_manager.cc:194] Registered new tserver with Master: f6db82e7e7554c0290a1c6a9e4b692cf (127.21.228.195:35067)
02:06:33.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:33.470598 23136 heartbeater.cc:347] Connected to a master server at 127.21.228.254:43509
02:06:33.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:33.470729 22617 ts_manager.cc:194] Registered new tserver with Master: f6db82e7e7554c0290a1c6a9e4b692cf (127.21.228.195:35067)
02:06:33.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:33.470952 23136 heartbeater.cc:464] Registering TS with master...
02:06:33.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:33.471686 23136 heartbeater.cc:511] Master 127.21.228.254:43509 requested a full tablet report, sending...
02:06:33.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:33.472879 22617 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.21.228.195:60177
02:06:33.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:33.474094 22464 ts_manager.cc:194] Registered new tserver with Master: f6db82e7e7554c0290a1c6a9e4b692cf (127.21.228.195:35067)
02:06:33.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:33.491370 22419 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
02:06:33.503 [DEBUG - main] (MiniKuduCluster.java:195) Response:
02:06:33.504 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
02:06:33.515 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.21.228.254"
port: 43509
}
bound_http_address {
host: "127.21.228.254"
port: 40939
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.21.228.253"
port: 35519
}
bound_http_address {
host: "127.21.228.253"
port: 35173
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.21.228.252"
port: 39791
}
bound_http_address {
host: "127.21.228.252"
port: 33187
}
}
}
02:06:33.520 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
02:06:33.523 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.21.228.193"
port: 41319
}
bound_http_address {
host: "127.21.228.193"
port: 37675
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.21.228.194"
port: 33523
}
bound_http_address {
host: "127.21.228.194"
port: 46833
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.21.228.195"
port: 35067
}
bound_http_address {
host: "127.21.228.195"
port: 37521
}
}
}
02:06:33.523 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
02:06:33.735 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
02:06:33.858 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskQ73J9W/test-tmp
02:06:33.858 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
02:06:33.859 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskQ73J9W/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
02:06:33.862 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
02:06:34.196 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:06:34.197 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
02:06:34.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:34.200073 23145 mini_oidc.cc:152] Starting JWKS server
02:06:34.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:34.203560 23145 webserver.cc:460] Webserver started at http://127.0.0.1:34521/ using document root <none> and password file <none>
02:06:34.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:34.204463 23145 mini_oidc.cc:189] Starting OIDC Discovery server
02:06:34.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:34.206749 23145 webserver.cc:460] Webserver started at http://127.0.0.1:35767/ using document root <none> and password file <none>
02:06:34.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
02:06:34.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/krb5kdc/principal' for realm 'KRBTEST.COM',
02:06:34.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
02:06:34.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 02:06:34 dist-test-slave-nz2j krb5kdc[23158](info): setting up network...
02:06:34.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
02:06:34.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 02:06:34 dist-test-slave-nz2j krb5kdc[23158](info): set up 2 sockets
02:06:34.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 02:06:34 dist-test-slave-nz2j krb5kdc[23158](info): commencing operation
02:06:34.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
02:06:34.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:34.478147 23134 heartbeater.cc:503] Master 127.21.228.252:39791 was elected leader, sending a full tablet report...
02:06:35.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:35.386533 23145 mini_kdc.cc:121] Time spent starting KDC: real 1.153s user 0.003s sys 0.016s
02:06:35.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
02:06:35.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
02:06:35.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
02:06:35.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
02:06:35.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
02:06:35.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
02:06:35.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
02:06:35.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
02:06:35.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
02:06:35.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
02:06:35.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/krb5kdc/test-user.keytab.
02:06:35.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/krb5kdc/test-user.keytab.
02:06:35.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 02:06:35 dist-test-slave-nz2j krb5kdc[23158](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790388395, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:06:35.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
02:06:35.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-26T02:06:35Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
02:06:35.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-26T02:06:35Z Disabled control of system clock
02:06:35.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.22.154.126@KRBTEST.COM; defaulting to no policy
02:06:35.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:06:35.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.22.154.126@KRBTEST.COM" created.
02:06:35.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:06:35.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.22.154.126 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/krb5kdc/kudu_127.22.154.126.keytab.
02:06:35.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.22.154.126 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/krb5kdc/kudu_127.22.154.126.keytab.
02:06:35.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:35.584555 23145 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQ73J9W/build/tsan/bin/kudu
02:06:35.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQ73J9W/build/tsan/bin/kudu
02:06:35.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-0/wal
02:06:35.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-0/data
02:06:35.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:06:35.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:06:35.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:06:35.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:35.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:35.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:35.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-0/logs
02:06:35.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-0/data/info.pb
02:06:35.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:35.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:06:35.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:06:35.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:06:35.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:06:35.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:35.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:35.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:35.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:06:35.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:06:35.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:06:35.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:06:35.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.154.126:33485
02:06:35.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.154.126
02:06:35.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:06:35.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/krb5kdc/kudu_127.22.154.126.keytab
02:06:35.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.154.126
02:06:35.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:06:35.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:06:35.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:06:35.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.154.84:38957
02:06:35.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:35.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:35.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:35.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:06:35.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.154.126:33485,127.22.154.125:41045,127.22.154.124:43081
02:06:35.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:06:35.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34521/jwks.json
02:06:35.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:06:35.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:35.883316 23173 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:06:35.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:35.883848 23173 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:06:35.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:35.884531 23173 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:06:35.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:35.915633 23173 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:06:35.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:35.915926 23173 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:06:35.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:35.916281 23173 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:06:35.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:35.916616 23173 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:06:35.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:35.916941 23173 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:06:35.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:35.917263 23173 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:06:35.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:35.917510 23173 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34521/jwks.json
02:06:35.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:35.961141 23173 master_runner.cc:386] Master server non-default flags:
02:06:35.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:35.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.154.84:38957
02:06:35.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:35.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:35.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-0/data
02:06:35.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-0/wal
02:06:35.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:06:35.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:06:35.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.154.126:33485,127.22.154.125:41045,127.22.154.124:43081
02:06:35.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/krb5kdc/kudu_127.22.154.126.keytab
02:06:35.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.154.126
02:06:35.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:35.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:35.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:06:35.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.154.126:33485
02:06:35.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:06:35.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:06:35.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:06:35.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34521/jwks.json
02:06:35.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:06:35.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:06:35.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:06:35.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:35.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:35.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-0/data/info.pb
02:06:35.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.154.126
02:06:35.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:06:35.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:06:35.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:35.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:06:35.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:06:35.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:35.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-0/logs
02:06:35.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:35.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:06:35.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:06:35.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:06:35.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:06:35.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
02:06:35.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:06:35.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 00:58:27 UTC on ccycloud.quasar-hahlai.root.comops.site
02:06:35.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4673502
02:06:35.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:06:35.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:35.962204 23173 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:06:35.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:35.963120 23173 file_cache.cc:493] Constructed file cache file cache with capacity 419430
02:06:35.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:35.981200 23179 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:36.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:36.183708 22651 debug-util.cc:398] Leaking SignalData structure 0x7b0800072180 after lost signal to thread 22581
02:06:36.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:36.184294 22651 debug-util.cc:398] Leaking SignalData structure 0x7b08000c0b80 after lost signal to thread 22654
02:06:37.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:35.981253 23182 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:37.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:35.981436 23180 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:37.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:35.984282 23173 server_base.cc:1034] running on GCE node
02:06:37.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:37.247996 23173 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:06:37.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:37.251132 23173 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:06:37.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:37.252554 23173 hybrid_clock.cc:648] HybridClock initialized: now 1790388397252490 us; error 78 us; skew 500 ppm
02:06:37.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:37.253472 23173 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:06:37.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 02:06:37 dist-test-slave-nz2j krb5kdc[23158](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790388397, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.154.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:06:37.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:37.259786 23173 init.cc:377] Logged in from keytab as kudu/127.22.154.126@KRBTEST.COM (short username kudu)
02:06:37.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:37.265664 23173 webserver.cc:460] Webserver started at http://127.22.154.126:39279/ using document root <none> and password file <none>
02:06:37.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:37.266635 23173 fs_manager.cc:359] Metadata directory not provided
02:06:37.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:37.266889 23173 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:06:37.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:37.267438 23173 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
02:06:37.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:37.272739 23173 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-0/data/instance:
02:06:37.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "35f4ec22b53d435fb9e5ce86ca372e26"
02:06:37.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 02:06:37 on dist-test-slave-nz2j"
02:06:37.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:37.273912 23173 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-0/wal/instance:
02:06:37.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "35f4ec22b53d435fb9e5ce86ca372e26"
02:06:37.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 02:06:37 on dist-test-slave-nz2j"
02:06:37.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:37.281579 23173 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.000s sys 0.008s
02:06:37.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:37.288244 23190 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:06:37.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:37.289669 23173 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.003s sys 0.001s
02:06:37.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:37.290040 23173 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-0/data,/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-0/wal
02:06:37.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "35f4ec22b53d435fb9e5ce86ca372e26"
02:06:37.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 02:06:37 on dist-test-slave-nz2j"
02:06:37.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:37.290465 23173 fs_report.cc:383] FS layout report
02:06:37.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:06:37.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-0/wal
02:06:37.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-0/wal
02:06:37.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-0/data/data
02:06:37.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:06:37.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:06:37.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:06:37.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:06:37.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:06:37.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:06:37.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:06:37.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:06:37.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:06:37.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:06:37.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:06:37.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:37.361943 23173 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
02:06:37.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:37.367491 23173 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:06:37.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:37.367892 23173 kserver.cc:163] Server-wide thread pool size limit: 3276
02:06:37.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:37.475273 23173 rpc_server.cc:307] RPC server started. Bound to: 127.22.154.126:33485
02:06:37.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:37.475260 23241 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.22.154.126:33485 every 8 connection(s)
02:06:37.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:37.479413 23173 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-0/data/info.pb
02:06:37.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:37.484120 23145 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQ73J9W/build/tsan/bin/kudu as pid 23173
02:06:37.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:37.484827 23145 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-0/wal/instance
02:06:37.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:37.495939 23242 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:06:37.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.22.154.125@KRBTEST.COM; defaulting to no policy
02:06:37.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:06:37.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.22.154.125@KRBTEST.COM" created.
02:06:37.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:37.517570 23242 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.154.126" port: 33485 } has no permanent_uuid. Determining permanent_uuid...
02:06:37.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:06:37.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.22.154.125 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/krb5kdc/kudu_127.22.154.125.keytab.
02:06:37.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.22.154.125 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/krb5kdc/kudu_127.22.154.125.keytab.
02:06:37.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:37.546310 23145 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQ73J9W/build/tsan/bin/kudu
02:06:37.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQ73J9W/build/tsan/bin/kudu
02:06:37.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-1/wal
02:06:37.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-1/data
02:06:37.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:06:37.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:06:37.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:06:37.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:37.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:37.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:37.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-1/logs
02:06:37.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-1/data/info.pb
02:06:37.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:37.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:06:37.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:06:37.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:06:37.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:06:37.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:37.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:37.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:37.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:06:37.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:06:37.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:06:37.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:06:37.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.154.125:41045
02:06:37.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.154.125
02:06:37.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:06:37.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/krb5kdc/kudu_127.22.154.125.keytab
02:06:37.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.154.125
02:06:37.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:06:37.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:06:37.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:06:37.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.154.84:38957
02:06:37.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:37.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:37.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:37.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:06:37.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.154.126:33485,127.22.154.125:41045,127.22.154.124:43081
02:06:37.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:06:37.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34521/jwks.json
02:06:37.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:06:37.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 02:06:37 dist-test-slave-nz2j krb5kdc[23158](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790388397, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.154.126@KRBTEST.COM for kudu/127.22.154.126@KRBTEST.COM
02:06:37.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:37.585574 23242 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.154.125" port: 41045 } has no permanent_uuid. Determining permanent_uuid...
02:06:37.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:37.588917 23191 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.154.125:41045: connect: Connection refused (error 111)
02:06:37.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:37.593096 23242 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.154.125:41045: Network error: Client connection negotiation failed: client connection to 127.22.154.125:41045: connect: Connection refused (error 111)
02:06:37.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:37.642603 23242 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.154.125" port: 41045 } attempt: 1
02:06:37.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:37.647125 23242 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.154.125:41045: Network error: Client connection negotiation failed: client connection to 127.22.154.125:41045: connect: Connection refused (error 111)
02:06:37.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:37.715574 23242 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.154.125" port: 41045 } attempt: 2
02:06:37.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:37.720371 23242 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.154.125:41045: Network error: Client connection negotiation failed: client connection to 127.22.154.125:41045: connect: Connection refused (error 111)
02:06:37.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:37.811823 23242 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.154.125" port: 41045 } attempt: 3
02:06:37.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:37.816596 23242 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.154.125:41045: Network error: Client connection negotiation failed: client connection to 127.22.154.125:41045: connect: Connection refused (error 111)
02:06:37.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:37.875981 23248 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:06:37.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:37.876617 23248 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:06:37.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:37.877182 23248 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:06:37.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:37.908810 23248 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:06:37.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:37.909029 23248 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:06:37.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:37.909289 23248 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:06:37.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:37.909521 23248 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:06:37.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:37.909713 23248 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:06:37.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:37.909982 23248 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:06:37.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:37.910140 23248 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34521/jwks.json
02:06:37.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:37.948724 23248 master_runner.cc:386] Master server non-default flags:
02:06:37.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:37.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.154.84:38957
02:06:37.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:37.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:37.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-1/data
02:06:37.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-1/wal
02:06:37.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:06:37.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:06:37.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.154.126:33485,127.22.154.125:41045,127.22.154.124:43081
02:06:37.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/krb5kdc/kudu_127.22.154.125.keytab
02:06:37.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.154.125
02:06:37.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:37.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:37.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:06:37.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.154.125:41045
02:06:37.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:06:37.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:06:37.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:06:37.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34521/jwks.json
02:06:37.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:06:37.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:06:37.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:06:37.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:37.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:37.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-1/data/info.pb
02:06:37.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.154.125
02:06:37.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:06:37.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:06:37.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:37.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:06:37.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:06:37.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:37.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-1/logs
02:06:37.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:37.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:06:37.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:06:37.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:06:37.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:06:37.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
02:06:37.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:06:37.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 00:58:27 UTC on ccycloud.quasar-hahlai.root.comops.site
02:06:37.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4673502
02:06:37.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:06:37.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:37.949822 23248 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:06:37.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:37.950953 23248 file_cache.cc:493] Constructed file cache file cache with capacity 419430
02:06:37.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:37.960041 23242 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.154.125" port: 41045 } attempt: 4
02:06:37.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:37.964636 23242 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.154.125:41045: Network error: Client connection negotiation failed: client connection to 127.22.154.125:41045: connect: Connection refused (error 111)
02:06:37.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:37.971235 23258 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:38.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:38.264055 23242 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.154.125" port: 41045 } attempt: 5
02:06:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:38.267804 23242 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.154.125:41045: Network error: Client connection negotiation failed: client connection to 127.22.154.125:41045: connect: Connection refused (error 111)
02:06:38.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:38.815384 23242 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.154.125" port: 41045 } attempt: 6
02:06:38.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:38.822253 23242 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.154.125:41045: Network error: Client connection negotiation failed: client connection to 127.22.154.125:41045: connect: Connection refused (error 111)
02:06:39.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:37.971230 23256 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:39.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:37.972828 23248 server_base.cc:1034] running on GCE node
02:06:39.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:37.972282 23255 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:39.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:39.210786 23248 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:06:39.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:39.214044 23248 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:06:39.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:39.215473 23248 hybrid_clock.cc:648] HybridClock initialized: now 1790388399215428 us; error 63 us; skew 500 ppm
02:06:39.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:39.216399 23248 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:06:39.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 02:06:39 dist-test-slave-nz2j krb5kdc[23158](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790388399, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.154.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:06:39.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:39.222337 23248 init.cc:377] Logged in from keytab as kudu/127.22.154.125@KRBTEST.COM (short username kudu)
02:06:39.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:39.228055 23248 webserver.cc:460] Webserver started at http://127.22.154.125:38269/ using document root <none> and password file <none>
02:06:39.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:39.229133 23248 fs_manager.cc:359] Metadata directory not provided
02:06:39.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:39.229326 23248 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:06:39.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:39.229693 23248 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
02:06:39.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:39.234520 23248 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-1/data/instance:
02:06:39.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5a3719d34abf44e088edadebf21b430c"
02:06:39.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 02:06:39 on dist-test-slave-nz2j"
02:06:39.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:39.235584 23248 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-1/wal/instance:
02:06:39.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5a3719d34abf44e088edadebf21b430c"
02:06:39.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 02:06:39 on dist-test-slave-nz2j"
02:06:39.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:39.244647 23248 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.007s sys 0.000s
02:06:39.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:39.252764 23270 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:06:39.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:39.254166 23248 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.000s sys 0.002s
02:06:39.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:39.254583 23248 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-1/data,/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-1/wal
02:06:39.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5a3719d34abf44e088edadebf21b430c"
02:06:39.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 02:06:39 on dist-test-slave-nz2j"
02:06:39.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:39.255081 23248 fs_report.cc:383] FS layout report
02:06:39.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:06:39.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-1/wal
02:06:39.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-1/wal
02:06:39.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-1/data/data
02:06:39.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:06:39.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:06:39.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:06:39.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:06:39.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:06:39.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:06:39.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:06:39.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:06:39.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:06:39.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:06:39.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:06:39.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:39.305274 23248 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
02:06:39.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:39.311120 23248 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:06:39.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:39.311633 23248 kserver.cc:163] Server-wide thread pool size limit: 3276
02:06:39.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:39.392589 23248 rpc_server.cc:307] RPC server started. Bound to: 127.22.154.125:41045
02:06:39.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:39.392640 23321 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.22.154.125:41045 every 8 connection(s)
02:06:39.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:39.395206 23248 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-1/data/info.pb
02:06:39.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:39.401019 23322 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:06:39.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:39.402671 23145 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQ73J9W/build/tsan/bin/kudu as pid 23248
02:06:39.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:39.403146 23145 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-1/wal/instance
02:06:39.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.22.154.124@KRBTEST.COM; defaulting to no policy
02:06:39.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:39.418498 23322 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.154.126" port: 33485 } has no permanent_uuid. Determining permanent_uuid...
02:06:39.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:06:39.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.22.154.124@KRBTEST.COM" created.
02:06:39.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:06:39.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.22.154.124 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/krb5kdc/kudu_127.22.154.124.keytab.
02:06:39.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.22.154.124 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/krb5kdc/kudu_127.22.154.124.keytab.
02:06:39.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:39.452469 23145 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQ73J9W/build/tsan/bin/kudu
02:06:39.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQ73J9W/build/tsan/bin/kudu
02:06:39.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-2/wal
02:06:39.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-2/data
02:06:39.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:06:39.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:06:39.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:06:39.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:39.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:39.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:39.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-2/logs
02:06:39.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-2/data/info.pb
02:06:39.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:39.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:06:39.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:06:39.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:06:39.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:06:39.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:39.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:39.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:39.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:06:39.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:06:39.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:06:39.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:06:39.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.154.124:43081
02:06:39.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.154.124
02:06:39.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:06:39.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/krb5kdc/kudu_127.22.154.124.keytab
02:06:39.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.154.124
02:06:39.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:06:39.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:06:39.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:06:39.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.154.84:38957
02:06:39.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:39.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:39.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:39.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:06:39.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.154.126:33485,127.22.154.125:41045,127.22.154.124:43081
02:06:39.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:06:39.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34521/jwks.json
02:06:39.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:06:39.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 02:06:39 dist-test-slave-nz2j krb5kdc[23158](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790388399, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.154.125@KRBTEST.COM for kudu/127.22.154.126@KRBTEST.COM
02:06:39.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:39.466869 23322 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.154.125" port: 41045 } has no permanent_uuid. Determining permanent_uuid...
02:06:39.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 02:06:39 dist-test-slave-nz2j krb5kdc[23158](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790388399, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.154.125@KRBTEST.COM for kudu/127.22.154.125@KRBTEST.COM
02:06:39.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:39.497694 23322 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.154.124" port: 43081 } has no permanent_uuid. Determining permanent_uuid...
02:06:39.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:39.501045 23271 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.154.124:43081: connect: Connection refused (error 111)
02:06:39.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:39.509747 23322 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.154.124:43081: Network error: Client connection negotiation failed: client connection to 127.22.154.124:43081: connect: Connection refused (error 111)
02:06:39.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:39.559268 23322 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.154.124" port: 43081 } attempt: 1
02:06:39.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:39.564947 23322 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.154.124:43081: Network error: Client connection negotiation failed: client connection to 127.22.154.124:43081: connect: Connection refused (error 111)
02:06:39.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:39.633360 23322 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.154.124" port: 43081 } attempt: 2
02:06:39.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:39.637437 23322 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.154.124:43081: Network error: Client connection negotiation failed: client connection to 127.22.154.124:43081: connect: Connection refused (error 111)
02:06:39.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:39.728834 23322 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.154.124" port: 43081 } attempt: 3
02:06:39.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:39.732457 23322 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.154.124:43081: Network error: Client connection negotiation failed: client connection to 127.22.154.124:43081: connect: Connection refused (error 111)
02:06:39.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:39.786903 23328 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:06:39.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:39.787598 23328 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:06:39.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:39.788390 23328 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:06:39.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:39.820420 23328 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:06:39.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:39.820693 23328 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:06:39.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:39.821079 23328 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:06:39.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:39.821404 23328 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:06:39.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:39.821703 23328 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:06:39.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:39.822010 23328 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:06:39.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:39.822242 23328 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34521/jwks.json
02:06:39.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:39.859359 23328 master_runner.cc:386] Master server non-default flags:
02:06:39.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:39.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.154.84:38957
02:06:39.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:39.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:39.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-2/data
02:06:39.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-2/wal
02:06:39.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:06:39.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:06:39.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.154.126:33485,127.22.154.125:41045,127.22.154.124:43081
02:06:39.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/krb5kdc/kudu_127.22.154.124.keytab
02:06:39.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.154.124
02:06:39.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:39.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:39.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:06:39.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.154.124:43081
02:06:39.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:06:39.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:06:39.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:06:39.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34521/jwks.json
02:06:39.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:06:39.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:06:39.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:06:39.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:39.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:39.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-2/data/info.pb
02:06:39.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.154.124
02:06:39.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:06:39.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:06:39.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:39.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:06:39.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:06:39.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:39.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-2/logs
02:06:39.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:39.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:06:39.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:06:39.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:06:39.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:06:39.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
02:06:39.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:06:39.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 00:58:27 UTC on ccycloud.quasar-hahlai.root.comops.site
02:06:39.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4673502
02:06:39.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:06:39.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:39.860886 23328 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:06:39.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:39.862028 23328 file_cache.cc:493] Constructed file cache file cache with capacity 419430
02:06:39.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:39.875945 23322 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.154.124" port: 43081 } attempt: 4
02:06:39.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:39.879664 23322 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.154.124:43081: Network error: Client connection negotiation failed: client connection to 127.22.154.124:43081: connect: Connection refused (error 111)
02:06:39.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:39.880817 23338 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:39.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:39.882823 23242 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.154.125" port: 41045 } attempt: 7
02:06:39.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 02:06:39 dist-test-slave-nz2j krb5kdc[23158](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790388397, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.154.126@KRBTEST.COM for kudu/127.22.154.125@KRBTEST.COM
02:06:39.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:39.916244 23242 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.154.124" port: 43081 } has no permanent_uuid. Determining permanent_uuid...
02:06:39.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:39.922739 23242 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.154.124:43081: Network error: Client connection negotiation failed: client connection to 127.22.154.124:43081: connect: Connection refused (error 111)
02:06:39.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:39.981093 23242 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.154.124" port: 43081 } attempt: 1
02:06:39.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:39.984714 23242 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.154.124:43081: Network error: Client connection negotiation failed: client connection to 127.22.154.124:43081: connect: Connection refused (error 111)
02:06:40.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:40.066356 23242 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.154.124" port: 43081 } attempt: 2
02:06:40.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:40.071784 23242 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.154.124:43081: Network error: Client connection negotiation failed: client connection to 127.22.154.124:43081: connect: Connection refused (error 111)
02:06:40.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:40.157167 23242 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.154.124" port: 43081 } attempt: 3
02:06:40.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:40.161484 23242 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.154.124:43081: Network error: Client connection negotiation failed: client connection to 127.22.154.124:43081: connect: Connection refused (error 111)
02:06:40.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:40.179076 23322 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.154.124" port: 43081 } attempt: 5
02:06:40.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:40.182643 23322 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.154.124:43081: Network error: Client connection negotiation failed: client connection to 127.22.154.124:43081: connect: Connection refused (error 111)
02:06:40.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:40.301887 23242 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.154.124" port: 43081 } attempt: 4
02:06:40.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:40.307088 23242 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.154.124:43081: Network error: Client connection negotiation failed: client connection to 127.22.154.124:43081: connect: Connection refused (error 111)
02:06:40.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:40.590492 23242 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.154.124" port: 43081 } attempt: 5
02:06:40.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:40.595258 23242 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.154.124:43081: Network error: Client connection negotiation failed: client connection to 127.22.154.124:43081: connect: Connection refused (error 111)
02:06:40.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:40.730237 23322 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.154.124" port: 43081 } attempt: 6
02:06:40.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:40.738643 23322 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.154.124:43081: Network error: Client connection negotiation failed: client connection to 127.22.154.124:43081: connect: Connection refused (error 111)
02:06:41.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:39.881126 23340 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:41.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:39.882531 23337 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:41.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:41.140803 23339 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
02:06:41.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:41.140945 23328 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:06:41.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:41.145246 23328 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:06:41.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:41.147835 23242 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.154.124" port: 43081 } attempt: 6
02:06:41.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:41.149240 23328 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:06:41.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:41.150795 23328 hybrid_clock.cc:648] HybridClock initialized: now 1790388401150714 us; error 77 us; skew 500 ppm
02:06:41.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:41.151790 23328 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:06:41.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:41.154762 23242 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.154.124:43081: Network error: Client connection negotiation failed: client connection to 127.22.154.124:43081: connect: Connection refused (error 111)
02:06:41.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 02:06:41 dist-test-slave-nz2j krb5kdc[23158](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790388401, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.154.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:06:41.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:41.158187 23328 init.cc:377] Logged in from keytab as kudu/127.22.154.124@KRBTEST.COM (short username kudu)
02:06:41.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:41.162853 23328 webserver.cc:460] Webserver started at http://127.22.154.124:46823/ using document root <none> and password file <none>
02:06:41.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:41.163754 23328 fs_manager.cc:359] Metadata directory not provided
02:06:41.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:41.163926 23328 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:06:41.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:41.164367 23328 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
02:06:41.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:41.168941 23328 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-2/data/instance:
02:06:41.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c39d2c906ca34275895e15883c59c78a"
02:06:41.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 02:06:41 on dist-test-slave-nz2j"
02:06:41.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:41.169938 23328 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-2/wal/instance:
02:06:41.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c39d2c906ca34275895e15883c59c78a"
02:06:41.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 02:06:41 on dist-test-slave-nz2j"
02:06:41.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:41.178442 23328 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.005s sys 0.003s
02:06:41.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:41.184787 23355 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:06:41.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:41.186197 23328 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.000s sys 0.004s
02:06:41.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:41.186496 23328 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-2/data,/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-2/wal
02:06:41.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c39d2c906ca34275895e15883c59c78a"
02:06:41.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 02:06:41 on dist-test-slave-nz2j"
02:06:41.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:41.186796 23328 fs_report.cc:383] FS layout report
02:06:41.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:06:41.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-2/wal
02:06:41.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-2/wal
02:06:41.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-2/data/data
02:06:41.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:06:41.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:06:41.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:06:41.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:06:41.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:06:41.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:06:41.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:06:41.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:06:41.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:06:41.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:06:41.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:06:41.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:41.227020 23328 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
02:06:41.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:41.232848 23328 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:06:41.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:41.233363 23328 kserver.cc:163] Server-wide thread pool size limit: 3276
02:06:41.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:41.331686 23328 rpc_server.cc:307] RPC server started. Bound to: 127.22.154.124:43081
02:06:41.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:41.331735 23406 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.22.154.124:43081 every 8 connection(s)
02:06:41.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:41.335381 23328 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-2/data/info.pb
02:06:41.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:41.341269 23145 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQ73J9W/build/tsan/bin/kudu as pid 23328
02:06:41.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:41.341730 23145 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-2/wal/instance
02:06:41.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:41.342425 23407 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:06:41.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:41.360700 23407 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.154.126" port: 33485 } has no permanent_uuid. Determining permanent_uuid...
02:06:41.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 02:06:41 dist-test-slave-nz2j krb5kdc[23158](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790388395, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.22.154.124@KRBTEST.COM
02:06:41.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 02:06:41 dist-test-slave-nz2j krb5kdc[23158](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790388401, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.154.124@KRBTEST.COM for kudu/127.22.154.126@KRBTEST.COM
02:06:41.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:41.399011 23407 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.154.125" port: 41045 } has no permanent_uuid. Determining permanent_uuid...
02:06:41.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 02:06:41 dist-test-slave-nz2j krb5kdc[23158](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790388401, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.154.124@KRBTEST.COM for kudu/127.22.154.125@KRBTEST.COM
02:06:41.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 02:06:41 dist-test-slave-nz2j krb5kdc[23158](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790388395, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.22.154.125@KRBTEST.COM
02:06:41.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:41.426437 23407 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.154.124" port: 43081 } has no permanent_uuid. Determining permanent_uuid...
02:06:41.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 02:06:41 dist-test-slave-nz2j krb5kdc[23158](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790388401, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.154.124@KRBTEST.COM for kudu/127.22.154.124@KRBTEST.COM
02:06:41.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 02:06:41 dist-test-slave-nz2j krb5kdc[23158](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790388395, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.22.154.126@KRBTEST.COM
02:06:41.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:41.456362 22495 debug-util.cc:398] Leaking SignalData structure 0x7b08000a9f40 after lost signal to thread 22432
02:06:41.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:41.457103 22495 debug-util.cc:398] Leaking SignalData structure 0x7b0800089ac0 after lost signal to thread 22498
02:06:41.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:41.461064 23407 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a: Bootstrap starting.
02:06:41.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:41.467275 23145 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
02:06:41.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:41.470512 23407 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a: Neither blocks nor log segments found. Creating new log.
02:06:41.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:41.473016 23407 log.cc:824] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a: Log is configured to *not* fsync() on all Append() calls
02:06:41.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:41.481271 23407 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a: No bootstrap required, opened a new log
02:06:41.490 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:06:41.491 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
02:06:41.512 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
02:06:41.513 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
02:06:41.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:41.516486 23407 raft_consensus.cc:348] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "35f4ec22b53d435fb9e5ce86ca372e26" member_type: VOTER last_known_addr { host: "127.22.154.126" port: 33485 } } peers { permanent_uuid: "5a3719d34abf44e088edadebf21b430c" member_type: VOTER last_known_addr { host: "127.22.154.125" port: 41045 } } peers { permanent_uuid: "c39d2c906ca34275895e15883c59c78a" member_type: VOTER last_known_addr { host: "127.22.154.124" port: 43081 } }
02:06:41.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:41.517527 23407 raft_consensus.cc:374] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:06:41.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:41.517844 23407 raft_consensus.cc:729] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c39d2c906ca34275895e15883c59c78a, State: Initialized, Role: FOLLOWER
02:06:41.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:41.518885 23407 consensus_queue.cc:260] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "35f4ec22b53d435fb9e5ce86ca372e26" member_type: VOTER last_known_addr { host: "127.22.154.126" port: 33485 } } peers { permanent_uuid: "5a3719d34abf44e088edadebf21b430c" member_type: VOTER last_known_addr { host: "127.22.154.125" port: 41045 } } peers { permanent_uuid: "c39d2c906ca34275895e15883c59c78a" member_type: VOTER last_known_addr { host: "127.22.154.124" port: 43081 } }
02:06:41.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:41.522318 23416 sys_catalog.cc:455] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "35f4ec22b53d435fb9e5ce86ca372e26" member_type: VOTER last_known_addr { host: "127.22.154.126" port: 33485 } } peers { permanent_uuid: "5a3719d34abf44e088edadebf21b430c" member_type: VOTER last_known_addr { host: "127.22.154.125" port: 41045 } } peers { permanent_uuid: "c39d2c906ca34275895e15883c59c78a" member_type: VOTER last_known_addr { host: "127.22.154.124" port: 43081 } } }
02:06:41.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:41.523453 23416 sys_catalog.cc:458] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a [sys.catalog]: This master's current role is: FOLLOWER
02:06:41.523 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.22.154.126"
port: 33485
}
bound_http_address {
host: "127.22.154.126"
port: 39279
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.22.154.125"
port: 41045
}
bound_http_address {
host: "127.22.154.125"
port: 38269
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.22.154.124"
port: 43081
}
bound_http_address {
host: "127.22.154.124"
port: 46823
}
}
}
02:06:41.524 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
02:06:41.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:41.524089 23407 sys_catalog.cc:564] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a [sys.catalog]: configured and running, proceeding with master startup.
02:06:41.524 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:06:41.525 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
02:06:41.526 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
02:06:41.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 02:06:41 dist-test-slave-nz2j krb5kdc[23158](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790388401, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:06:41.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
02:06:41.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:41.576063 23428 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:06:41.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:41.584462 23428 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:06:41.595 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:06:41.598 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/krb5kdc/krb5cc
02:06:41.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:41.560520 23328 master_runner.cc:186] Error getting master registration for 127.22.154.126:33485: OK, instance_id { permanent_uuid: "35f4ec22b53d435fb9e5ce86ca372e26" instance_seqno: 1790388397448780 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
02:06:41.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:41.598654 23328 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:06:41.610 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
02:06:41.612 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/krb5kdc/krb5cc
02:06:41.617 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
02:06:41.621 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
02:06:41.623 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
02:06:41.624 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
02:06:41.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 02:06:41 dist-test-slave-nz2j krb5kdc[23158](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790388401, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:06:41.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
02:06:41.693 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:06:41.694 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/krb5kdc/krb5cc
02:06:41.695 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
02:06:41.696 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.22.154.126:33485
02:06:41.698 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
02:06:41.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:41.698951 23145 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskQ73J9W/build/tsan/bin/kudu with pid 23173
02:06:41.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:41.742440 23361 tablet.cc:2369] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:06:41.744 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:06:41.745 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.22.154.124:43081
02:06:41.745 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
02:06:41.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:41.745643 23145 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskQ73J9W/build/tsan/bin/kudu with pid 23328
02:06:41.775 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:06:41.775 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.22.154.125:41045
02:06:41.776 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
02:06:41.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:41.776292 23145 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskQ73J9W/build/tsan/bin/kudu with pid 23248
02:06:41.799 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:06:41.800 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.22.154.126:33485
02:06:41.802 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
02:06:41.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:41.803807 23145 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQ73J9W/build/tsan/bin/kudu
02:06:41.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQ73J9W/build/tsan/bin/kudu
02:06:41.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-0/wal
02:06:41.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-0/data
02:06:41.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:06:41.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:06:41.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:06:41.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:41.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:41.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:41.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-0/logs
02:06:41.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-0/data/info.pb
02:06:41.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:41.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:06:41.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:06:41.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:06:41.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:06:41.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:41.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:41.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:41.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:06:41.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:06:41.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:06:41.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:06:41.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.154.126:33485
02:06:41.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.154.126
02:06:41.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39279
02:06:41.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/krb5kdc/kudu_127.22.154.126.keytab
02:06:41.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.154.126
02:06:41.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:06:41.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:06:41.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:06:41.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.154.84:38957
02:06:41.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:41.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:41.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:41.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:06:41.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.154.126:33485,127.22.154.125:41045,127.22.154.124:43081
02:06:41.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:06:41.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34521/jwks.json
02:06:41.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:06:42.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:42.142300 23434 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:06:42.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:42.142968 23434 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:06:42.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:42.143764 23434 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:06:42.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:42.177169 23434 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:06:42.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:42.177438 23434 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:06:42.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:42.177830 23434 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:06:42.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:42.178164 23434 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:06:42.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:42.178476 23434 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:06:42.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:42.178808 23434 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:06:42.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:42.179029 23434 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34521/jwks.json
02:06:42.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:42.218364 23434 master_runner.cc:386] Master server non-default flags:
02:06:42.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:42.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.154.84:38957
02:06:42.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:42.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:42.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-0/data
02:06:42.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-0/wal
02:06:42.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:06:42.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:06:42.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.154.126:33485,127.22.154.125:41045,127.22.154.124:43081
02:06:42.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/krb5kdc/kudu_127.22.154.126.keytab
02:06:42.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.154.126
02:06:42.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:42.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:42.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:06:42.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.154.126:33485
02:06:42.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:06:42.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:06:42.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:06:42.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34521/jwks.json
02:06:42.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:06:42.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:06:42.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:06:42.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:42.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:42.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-0/data/info.pb
02:06:42.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.154.126
02:06:42.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39279
02:06:42.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:06:42.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:42.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:06:42.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:06:42.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:42.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-0/logs
02:06:42.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:42.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:06:42.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:06:42.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:06:42.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:06:42.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
02:06:42.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:06:42.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 00:58:27 UTC on ccycloud.quasar-hahlai.root.comops.site
02:06:42.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4673502
02:06:42.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:06:42.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:42.219908 23434 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:06:42.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:42.221652 23434 file_cache.cc:493] Constructed file cache file cache with capacity 419430
02:06:42.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:42.245316 23440 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:43.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:42.246614 23441 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:43.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:42.256678 23443 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:43.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:43.512451 23442 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
02:06:43.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:43.513005 23434 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:06:43.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:43.516950 23434 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:06:43.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:43.519542 23434 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:06:43.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:43.520931 23434 hybrid_clock.cc:648] HybridClock initialized: now 1790388403520881 us; error 36 us; skew 500 ppm
02:06:43.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:43.521800 23434 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:06:43.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 02:06:43 dist-test-slave-nz2j krb5kdc[23158](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790388403, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.154.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:06:43.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:43.527782 23434 init.cc:377] Logged in from keytab as kudu/127.22.154.126@KRBTEST.COM (short username kudu)
02:06:43.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:43.532860 23434 webserver.cc:460] Webserver started at http://127.22.154.126:39279/ using document root <none> and password file <none>
02:06:43.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:43.533697 23434 fs_manager.cc:359] Metadata directory not provided
02:06:43.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:43.533885 23434 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:06:43.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:43.541980 23434 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.000s sys 0.004s
02:06:43.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:43.546015 23452 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:06:43.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:43.547137 23434 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
02:06:43.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:43.547407 23434 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-0/data,/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-0/wal
02:06:43.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "35f4ec22b53d435fb9e5ce86ca372e26"
02:06:43.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 02:06:37 on dist-test-slave-nz2j"
02:06:43.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:43.549731 23434 fs_report.cc:383] FS layout report
02:06:43.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:06:43.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-0/wal
02:06:43.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-0/wal
02:06:43.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-0/data/data
02:06:43.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:06:43.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:06:43.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:06:43.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:06:43.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:06:43.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:06:43.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:06:43.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:06:43.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:06:43.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:06:43.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:06:43.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:43.601291 23434 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
02:06:43.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:43.606957 23434 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:06:43.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:43.607436 23434 kserver.cc:163] Server-wide thread pool size limit: 3276
02:06:43.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:43.678318 23434 rpc_server.cc:307] RPC server started. Bound to: 127.22.154.126:33485
02:06:43.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:43.678453 23503 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.22.154.126:33485 every 8 connection(s)
02:06:43.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:43.681011 23434 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-0/data/info.pb
02:06:43.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:43.684527 23145 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQ73J9W/build/tsan/bin/kudu as pid 23434
02:06:43.685 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:06:43.685 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.22.154.124:43081
02:06:43.685 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
02:06:43.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:43.687037 23145 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQ73J9W/build/tsan/bin/kudu
02:06:43.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQ73J9W/build/tsan/bin/kudu
02:06:43.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-2/wal
02:06:43.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-2/data
02:06:43.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:06:43.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:06:43.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:06:43.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:43.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:43.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:43.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-2/logs
02:06:43.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-2/data/info.pb
02:06:43.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:43.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:06:43.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:06:43.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:06:43.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:06:43.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:43.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:43.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:43.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:06:43.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:06:43.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:06:43.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:06:43.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.154.124:43081
02:06:43.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.154.124
02:06:43.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46823
02:06:43.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/krb5kdc/kudu_127.22.154.124.keytab
02:06:43.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.154.124
02:06:43.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:06:43.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:06:43.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:06:43.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.154.84:38957
02:06:43.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:43.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:43.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:43.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:06:43.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.154.126:33485,127.22.154.125:41045,127.22.154.124:43081
02:06:43.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:06:43.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34521/jwks.json
02:06:43.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:06:43.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:43.693396 23504 sys_catalog.cc:263] Verifying existing consensus state
02:06:43.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260926 02:06:43.694182 23504 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
02:06:43.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:43.694613 23434 master.cc:537] Master@127.22.154.126:33485 shutting down...
02:06:43.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:43.757382 23434 master.cc:552] Master@127.22.154.126:33485 shutdown complete.
02:06:43.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
02:06:44.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:44.011312 23506 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:06:44.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:44.011989 23506 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:06:44.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:44.012760 23506 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:06:44.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:44.045207 23506 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:06:44.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:44.045481 23506 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:06:44.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:44.045856 23506 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:06:44.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:44.046164 23506 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:06:44.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:44.046486 23506 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:06:44.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:44.046798 23506 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:06:44.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:44.047041 23506 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34521/jwks.json
02:06:44.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:44.083612 23506 master_runner.cc:386] Master server non-default flags:
02:06:44.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:44.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.154.84:38957
02:06:44.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:44.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:44.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-2/data
02:06:44.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-2/wal
02:06:44.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:06:44.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:06:44.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.154.126:33485,127.22.154.125:41045,127.22.154.124:43081
02:06:44.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/krb5kdc/kudu_127.22.154.124.keytab
02:06:44.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.154.124
02:06:44.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:44.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:44.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:06:44.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.154.124:43081
02:06:44.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:06:44.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:06:44.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:06:44.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34521/jwks.json
02:06:44.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:06:44.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:06:44.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:06:44.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:44.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:44.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-2/data/info.pb
02:06:44.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.154.124
02:06:44.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46823
02:06:44.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:06:44.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:44.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:06:44.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:06:44.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:44.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-2/logs
02:06:44.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:44.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:06:44.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:06:44.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:06:44.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:06:44.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
02:06:44.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:06:44.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 00:58:27 UTC on ccycloud.quasar-hahlai.root.comops.site
02:06:44.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4673502
02:06:44.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:06:44.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:44.085080 23506 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:06:44.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:44.086440 23506 file_cache.cc:493] Constructed file cache file cache with capacity 419430
02:06:45.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:44.106529 23506 server_base.cc:1034] running on GCE node
02:06:45.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:44.105032 23512 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:45.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:44.104990 23515 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:45.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:44.105000 23513 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:45.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:45.353529 23506 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:06:45.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:45.356588 23506 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:06:45.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:45.358069 23506 hybrid_clock.cc:648] HybridClock initialized: now 1790388405358024 us; error 61 us; skew 500 ppm
02:06:45.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:45.358909 23506 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:06:45.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 02:06:45 dist-test-slave-nz2j krb5kdc[23158](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790388405, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.154.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:06:45.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:45.365342 23506 init.cc:377] Logged in from keytab as kudu/127.22.154.124@KRBTEST.COM (short username kudu)
02:06:45.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:45.371127 23506 webserver.cc:460] Webserver started at http://127.22.154.124:46823/ using document root <none> and password file <none>
02:06:45.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:45.372110 23506 fs_manager.cc:359] Metadata directory not provided
02:06:45.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:45.372442 23506 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:06:45.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:45.380975 23506 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.004s sys 0.003s
02:06:45.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:45.385699 23524 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:06:45.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:45.386983 23506 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.003s
02:06:45.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:45.387254 23506 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-2/data,/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-2/wal
02:06:45.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c39d2c906ca34275895e15883c59c78a"
02:06:45.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 02:06:41 on dist-test-slave-nz2j"
02:06:45.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:45.389693 23506 fs_report.cc:383] FS layout report
02:06:45.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:06:45.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-2/wal
02:06:45.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-2/wal
02:06:45.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-2/data/data
02:06:45.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:06:45.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:06:45.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:06:45.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:06:45.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:06:45.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:06:45.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:06:45.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:06:45.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:06:45.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:06:45.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:06:45.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:45.442066 23506 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
02:06:45.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:45.447824 23506 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:06:45.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:45.448275 23506 kserver.cc:163] Server-wide thread pool size limit: 3276
02:06:45.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:45.530352 23506 rpc_server.cc:307] RPC server started. Bound to: 127.22.154.124:43081
02:06:45.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:45.530444 23576 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.22.154.124:43081 every 8 connection(s)
02:06:45.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:45.533416 23506 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-2/data/info.pb
02:06:45.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:45.541718 23145 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQ73J9W/build/tsan/bin/kudu as pid 23506
02:06:45.543 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:06:45.543 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.22.154.125:41045
02:06:45.543 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
02:06:45.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:45.544788 23145 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQ73J9W/build/tsan/bin/kudu
02:06:45.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQ73J9W/build/tsan/bin/kudu
02:06:45.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-1/wal
02:06:45.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-1/data
02:06:45.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:06:45.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:06:45.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:06:45.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:45.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:45.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:45.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-1/logs
02:06:45.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-1/data/info.pb
02:06:45.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:45.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:06:45.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:06:45.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:06:45.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:06:45.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:45.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:45.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:45.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:06:45.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:06:45.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:06:45.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:06:45.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.154.125:41045
02:06:45.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.154.125
02:06:45.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38269
02:06:45.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/krb5kdc/kudu_127.22.154.125.keytab
02:06:45.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.154.125
02:06:45.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:06:45.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:06:45.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:06:45.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.154.84:38957
02:06:45.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:45.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:45.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:45.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:06:45.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.154.126:33485,127.22.154.125:41045,127.22.154.124:43081
02:06:45.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:06:45.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34521/jwks.json
02:06:45.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:06:45.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:45.544935 23577 sys_catalog.cc:263] Verifying existing consensus state
02:06:45.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:45.551827 23577 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a: Bootstrap starting.
02:06:45.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:45.585023 23577 log.cc:824] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a: Log is configured to *not* fsync() on all Append() calls
02:06:45.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:45.589448 23577 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
02:06:45.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:45.590058 23577 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a: Bootstrap complete.
02:06:45.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:45.614198 23577 raft_consensus.cc:348] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "35f4ec22b53d435fb9e5ce86ca372e26" member_type: VOTER last_known_addr { host: "127.22.154.126" port: 33485 } } peers { permanent_uuid: "5a3719d34abf44e088edadebf21b430c" member_type: VOTER last_known_addr { host: "127.22.154.125" port: 41045 } } peers { permanent_uuid: "c39d2c906ca34275895e15883c59c78a" member_type: VOTER last_known_addr { host: "127.22.154.124" port: 43081 } }
02:06:45.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:45.616570 23577 raft_consensus.cc:374] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:06:45.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:45.616794 23577 raft_consensus.cc:729] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c39d2c906ca34275895e15883c59c78a, State: Initialized, Role: FOLLOWER
02:06:45.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:45.617571 23577 consensus_queue.cc:260] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "35f4ec22b53d435fb9e5ce86ca372e26" member_type: VOTER last_known_addr { host: "127.22.154.126" port: 33485 } } peers { permanent_uuid: "5a3719d34abf44e088edadebf21b430c" member_type: VOTER last_known_addr { host: "127.22.154.125" port: 41045 } } peers { permanent_uuid: "c39d2c906ca34275895e15883c59c78a" member_type: VOTER last_known_addr { host: "127.22.154.124" port: 43081 } }
02:06:45.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:45.620465 23581 sys_catalog.cc:455] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "35f4ec22b53d435fb9e5ce86ca372e26" member_type: VOTER last_known_addr { host: "127.22.154.126" port: 33485 } } peers { permanent_uuid: "5a3719d34abf44e088edadebf21b430c" member_type: VOTER last_known_addr { host: "127.22.154.125" port: 41045 } } peers { permanent_uuid: "c39d2c906ca34275895e15883c59c78a" member_type: VOTER last_known_addr { host: "127.22.154.124" port: 43081 } } }
02:06:45.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:45.621326 23581 sys_catalog.cc:458] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a [sys.catalog]: This master's current role is: FOLLOWER
02:06:45.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:45.622071 23577 sys_catalog.cc:564] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a [sys.catalog]: configured and running, proceeding with master startup.
02:06:45.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:45.652064 23592 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:06:45.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:45.652455 23592 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:06:45.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:45.649420 23506 master_runner.cc:186] Error getting master registration for 127.22.154.126:33485: Network error: Client connection negotiation failed: client connection to 127.22.154.126:33485: connect: Connection refused (error 111),
02:06:45.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:45.669278 23506 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:06:45.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:45.703420 23530 tablet.cc:2369] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:06:45.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:45.707051 23581 raft_consensus.cc:482] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:06:45.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:45.707583 23581 raft_consensus.cc:504] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "35f4ec22b53d435fb9e5ce86ca372e26" member_type: VOTER last_known_addr { host: "127.22.154.126" port: 33485 } } peers { permanent_uuid: "5a3719d34abf44e088edadebf21b430c" member_type: VOTER last_known_addr { host: "127.22.154.125" port: 41045 } } peers { permanent_uuid: "c39d2c906ca34275895e15883c59c78a" member_type: VOTER last_known_addr { host: "127.22.154.124" port: 43081 } }
02:06:45.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:45.711313 23581 leader_election.cc:288] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 35f4ec22b53d435fb9e5ce86ca372e26 (127.22.154.126:33485), 5a3719d34abf44e088edadebf21b430c (127.22.154.125:41045)
02:06:45.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:45.714095 23527 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.154.126:33485: connect: Connection refused (error 111) [suppressed 1 similar messages]
02:06:45.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:45.723443 23525 leader_election.cc:334] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 5a3719d34abf44e088edadebf21b430c (127.22.154.125:41045): Network error: Client connection negotiation failed: client connection to 127.22.154.125:41045: connect: Connection refused (error 111)
02:06:45.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:45.724128 23527 leader_election.cc:334] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 35f4ec22b53d435fb9e5ce86ca372e26 (127.22.154.126:33485): Network error: Client connection negotiation failed: client connection to 127.22.154.126:33485: connect: Connection refused (error 111)
02:06:45.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:45.724565 23527 leader_election.cc:302] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: c39d2c906ca34275895e15883c59c78a; no voters: 35f4ec22b53d435fb9e5ce86ca372e26, 5a3719d34abf44e088edadebf21b430c
02:06:45.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:45.725850 23581 raft_consensus.cc:2726] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
02:06:45.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:45.883195 23579 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:06:45.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:45.883853 23579 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:06:45.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:45.884622 23579 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:06:45.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:45.918418 23579 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:06:45.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:45.918700 23579 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:06:45.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:45.919080 23579 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:06:45.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:45.919413 23579 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:06:45.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:45.919718 23579 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:06:45.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:45.920019 23579 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:06:45.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:45.920269 23579 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34521/jwks.json
02:06:45.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:45.958912 23579 master_runner.cc:386] Master server non-default flags:
02:06:45.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:45.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.154.84:38957
02:06:45.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:45.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:45.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-1/data
02:06:45.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-1/wal
02:06:45.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:06:45.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:06:45.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.154.126:33485,127.22.154.125:41045,127.22.154.124:43081
02:06:45.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/krb5kdc/kudu_127.22.154.125.keytab
02:06:45.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.154.125
02:06:45.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:45.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:45.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:06:45.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.154.125:41045
02:06:45.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:06:45.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:06:45.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:06:45.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34521/jwks.json
02:06:45.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:06:45.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:06:45.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:06:45.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:45.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:45.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-1/data/info.pb
02:06:45.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.154.125
02:06:45.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38269
02:06:45.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:06:45.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:45.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:06:45.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:06:45.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:45.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-1/logs
02:06:45.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:45.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:06:45.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:06:45.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:06:45.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:06:45.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
02:06:45.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:06:45.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 00:58:27 UTC on ccycloud.quasar-hahlai.root.comops.site
02:06:45.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4673502
02:06:45.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:06:45.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:45.960428 23579 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:06:45.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:45.961717 23579 file_cache.cc:493] Constructed file cache file cache with capacity 419430
02:06:45.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:45.981434 23605 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:46.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:46.544720 23573 messenger.cc:188] Invalid argument: socket address is ambiguous: no data on pending connections for acceptor pool at 127.22.154.124:43081
02:06:46.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:46.658895 23592 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:06:46.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:46.659402 23592 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:06:47.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:45.984763 23602 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:47.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:47.301492 23604 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
02:06:47.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:47.302062 23579 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:06:47.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:45.981458 23603 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:47.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:47.305783 23579 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:06:47.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:47.309798 23579 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:06:47.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:47.311241 23579 hybrid_clock.cc:648] HybridClock initialized: now 1790388407311195 us; error 51 us; skew 500 ppm
02:06:47.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:47.312335 23579 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:06:47.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 02:06:47 dist-test-slave-nz2j krb5kdc[23158](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790388407, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.154.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:06:47.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:47.320298 23579 init.cc:377] Logged in from keytab as kudu/127.22.154.125@KRBTEST.COM (short username kudu)
02:06:47.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:47.346791 23579 webserver.cc:460] Webserver started at http://127.22.154.125:38269/ using document root <none> and password file <none>
02:06:47.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:47.347918 23579 fs_manager.cc:359] Metadata directory not provided
02:06:47.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:47.348260 23579 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:06:47.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:47.356683 23579 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.004s sys 0.000s
02:06:47.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:47.361522 23613 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:06:47.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:47.362680 23579 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
02:06:47.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:47.363035 23579 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-1/data,/tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-1/wal
02:06:47.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5a3719d34abf44e088edadebf21b430c"
02:06:47.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 02:06:39 on dist-test-slave-nz2j"
02:06:47.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:47.365339 23579 fs_report.cc:383] FS layout report
02:06:47.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:06:47.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-1/wal
02:06:47.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-1/wal
02:06:47.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-1/data/data
02:06:47.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:06:47.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:06:47.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:06:47.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:06:47.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:06:47.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:06:47.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:06:47.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:06:47.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:06:47.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:06:47.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:06:47.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:47.406138 23579 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
02:06:47.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:47.412900 23579 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:06:47.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:47.413417 23579 kserver.cc:163] Server-wide thread pool size limit: 3276
02:06:47.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:47.468221 23637 raft_consensus.cc:482] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:06:47.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:47.469260 23637 raft_consensus.cc:504] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "35f4ec22b53d435fb9e5ce86ca372e26" member_type: VOTER last_known_addr { host: "127.22.154.126" port: 33485 } } peers { permanent_uuid: "5a3719d34abf44e088edadebf21b430c" member_type: VOTER last_known_addr { host: "127.22.154.125" port: 41045 } } peers { permanent_uuid: "c39d2c906ca34275895e15883c59c78a" member_type: VOTER last_known_addr { host: "127.22.154.124" port: 43081 } }
02:06:47.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:47.474491 23637 leader_election.cc:288] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 35f4ec22b53d435fb9e5ce86ca372e26 (127.22.154.126:33485), 5a3719d34abf44e088edadebf21b430c (127.22.154.125:41045)
02:06:47.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:47.491989 23525 leader_election.cc:334] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 5a3719d34abf44e088edadebf21b430c (127.22.154.125:41045): Network error: Client connection negotiation failed: client connection to 127.22.154.125:41045: connect: Connection refused (error 111)
02:06:47.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:47.493542 23527 leader_election.cc:334] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 35f4ec22b53d435fb9e5ce86ca372e26 (127.22.154.126:33485): Network error: Client connection negotiation failed: client connection to 127.22.154.126:33485: connect: Connection refused (error 111)
02:06:47.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:47.493876 23527 leader_election.cc:302] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: c39d2c906ca34275895e15883c59c78a; no voters: 35f4ec22b53d435fb9e5ce86ca372e26, 5a3719d34abf44e088edadebf21b430c
02:06:47.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:47.494684 23637 raft_consensus.cc:2726] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
02:06:47.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:47.537092 23579 rpc_server.cc:307] RPC server started. Bound to: 127.22.154.125:41045
02:06:47.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:47.537309 23668 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.22.154.125:41045 every 8 connection(s)
02:06:47.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:47.540879 23579 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-1/data/info.pb
02:06:47.542 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:06:47.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:47.541442 23145 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQ73J9W/build/tsan/bin/kudu as pid 23579
02:06:47.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:47.562804 23669 sys_catalog.cc:263] Verifying existing consensus state
02:06:47.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260926 02:06:47.563726 23669 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
02:06:47.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:47.564224 23579 master.cc:537] Master@127.22.154.125:41045 shutting down...
02:06:47.607 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.22.154.126:33485(127.22.154.126:33485)] connecting to peer
02:06:47.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:47.639115 23579 master.cc:552] Master@127.22.154.125:41045 shutdown complete.
02:06:47.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskQ73J9W/test-tmp/mini-kudu-cluster1130785859570937926/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
02:06:47.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:47.663569 23592 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:06:47.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:47.663901 23592 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:06:47.669 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.22.154.124:43081(127.22.154.124:43081)] connecting to peer
02:06:47.671 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.22.154.125:41045(127.22.154.125:41045)] connecting to peer
02:06:47.730 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.22.154.124:43081(127.22.154.124:43081)] Successfully connected to peer
02:06:47.733 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:06:47.733 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.154.126:33485(127.22.154.126:33485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:06:47.735 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.154.126:33485: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:06:47.737 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:06:47.737 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.154.125:41045(127.22.154.125:41045)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:06:47.738 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.154.125:41045: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:06:47.809 [WARN - kudu-nio-1] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
02:06:47.817 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.22.154.124
02:06:48.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 02:06:48 dist-test-slave-nz2j krb5kdc[23158](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790388401, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.22.154.124@KRBTEST.COM
02:06:48.044 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x69c82a55, L:/127.0.0.1:43202 - R:/127.22.154.124:43081] using SASL/GSSAPI
02:06:48.059 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045; will retry
02:06:48.068 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.154.126:33485(127.22.154.126:33485)] connecting to peer
02:06:48.073 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.154.125:41045(127.22.154.125:41045)] connecting to peer
02:06:48.075 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:06:48.075 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.154.125:41045(127.22.154.125:41045)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:06:48.076 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.154.125:41045: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:06:48.077 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:06:48.078 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.154.126:33485(127.22.154.126:33485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:06:48.079 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.154.126:33485: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:06:48.080 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045; will retry
02:06:48.088 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.154.126:33485(127.22.154.126:33485)] connecting to peer
02:06:48.089 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.154.125:41045(127.22.154.125:41045)] connecting to peer
02:06:48.094 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:06:48.094 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.154.125:41045(127.22.154.125:41045)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:06:48.094 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.154.125:41045: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:06:48.095 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:06:48.095 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.154.126:33485(127.22.154.126:33485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:06:48.096 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.154.126:33485: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:06:48.098 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045; will retry
02:06:48.107 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.154.126:33485(127.22.154.126:33485)] connecting to peer
02:06:48.109 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.154.125:41045(127.22.154.125:41045)] connecting to peer
02:06:48.111 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:06:48.111 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.154.125:41045(127.22.154.125:41045)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:06:48.113 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.154.125:41045: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:06:48.115 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:06:48.115 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.154.126:33485(127.22.154.126:33485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:06:48.116 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.154.126:33485: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:06:48.117 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045; will retry
02:06:48.128 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.154.126:33485(127.22.154.126:33485)] connecting to peer
02:06:48.130 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.154.125:41045(127.22.154.125:41045)] connecting to peer
02:06:48.131 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:06:48.132 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.154.125:41045(127.22.154.125:41045)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:06:48.132 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.154.125:41045: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:06:48.133 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:06:48.133 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.154.126:33485(127.22.154.126:33485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:06:48.133 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.154.126:33485: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:06:48.135 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045; will retry
02:06:48.168 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.154.126:33485(127.22.154.126:33485)] connecting to peer
02:06:48.169 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.154.125:41045(127.22.154.125:41045)] connecting to peer
02:06:48.171 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:06:48.171 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.154.125:41045(127.22.154.125:41045)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:06:48.172 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:06:48.172 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.154.126:33485(127.22.154.126:33485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:06:48.172 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.154.125:41045: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:06:48.173 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.154.126:33485: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:06:48.174 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045; will retry
02:06:48.248 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.154.126:33485(127.22.154.126:33485)] connecting to peer
02:06:48.250 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.154.125:41045(127.22.154.125:41045)] connecting to peer
02:06:48.251 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:06:48.252 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.154.125:41045(127.22.154.125:41045)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:06:48.252 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.154.125:41045: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:06:48.254 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:06:48.254 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.154.126:33485(127.22.154.126:33485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:06:48.255 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.154.126:33485: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:06:48.255 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045; will retry
02:06:48.368 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.154.126:33485(127.22.154.126:33485)] connecting to peer
02:06:48.370 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.154.125:41045(127.22.154.125:41045)] connecting to peer
02:06:48.372 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:06:48.372 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.154.126:33485(127.22.154.126:33485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:06:48.372 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:06:48.372 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.154.125:41045(127.22.154.125:41045)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:06:48.373 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.154.126:33485: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:06:48.373 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.154.125:41045: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:06:48.375 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045; will retry
02:06:48.508 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.154.126:33485(127.22.154.126:33485)] connecting to peer
02:06:48.509 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.154.125:41045(127.22.154.125:41045)] connecting to peer
02:06:48.511 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:06:48.511 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.154.125:41045(127.22.154.125:41045)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:06:48.512 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.154.125:41045: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:06:48.513 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:06:48.513 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.154.126:33485(127.22.154.126:33485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:06:48.513 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.154.126:33485: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:06:48.515 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045; will retry
02:06:48.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:48.668468 23592 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:06:48.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:48.668828 23592 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:06:48.748 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.154.126:33485(127.22.154.126:33485)] connecting to peer
02:06:48.750 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.154.125:41045(127.22.154.125:41045)] connecting to peer
02:06:48.752 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:06:48.752 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.154.125:41045(127.22.154.125:41045)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:06:48.752 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:06:48.752 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.154.126:33485(127.22.154.126:33485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:06:48.753 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.154.126:33485: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:06:48.753 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.154.125:41045: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:06:48.755 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045; will retry
02:06:48.928 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.154.126:33485(127.22.154.126:33485)] connecting to peer
02:06:48.929 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.154.125:41045(127.22.154.125:41045)] connecting to peer
02:06:48.931 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:06:48.931 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.154.125:41045(127.22.154.125:41045)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:06:48.931 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.154.125:41045: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:06:48.932 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:06:48.932 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.154.126:33485(127.22.154.126:33485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:06:48.933 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.154.126:33485: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:06:48.934 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045; will retry
02:06:49.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:49.671592 23592 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:06:49.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:49.671857 23592 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:06:50.287 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.154.126:33485(127.22.154.126:33485)] connecting to peer
02:06:50.290 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:06:50.290 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.154.126:33485(127.22.154.126:33485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:06:50.290 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.154.125:41045(127.22.154.125:41045)] connecting to peer
02:06:50.290 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.154.126:33485: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:06:50.292 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:06:50.293 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.154.125:41045(127.22.154.125:41045)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:06:50.293 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.154.125:41045: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:06:50.294 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045; will retry
02:06:50.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:50.583777 23573 debug-util.cc:398] Leaking SignalData structure 0x7b08000738c0 after lost signal to thread 23507
02:06:50.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:50.584429 23573 debug-util.cc:398] Leaking SignalData structure 0x7b08000ac440 after lost signal to thread 23576
02:06:50.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:50.674680 23592 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:06:50.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:50.674929 23592 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:06:50.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:50.688475 23506 master_runner.cc:186] Error getting master registration for 127.22.154.126:33485: Network error: Client connection negotiation failed: client connection to 127.22.154.126:33485: connect: Connection refused (error 111),
02:06:50.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:50.688961 23506 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:06:50.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:50.770121 23679 raft_consensus.cc:482] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:06:50.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:50.770560 23679 raft_consensus.cc:504] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "35f4ec22b53d435fb9e5ce86ca372e26" member_type: VOTER last_known_addr { host: "127.22.154.126" port: 33485 } } peers { permanent_uuid: "5a3719d34abf44e088edadebf21b430c" member_type: VOTER last_known_addr { host: "127.22.154.125" port: 41045 } } peers { permanent_uuid: "c39d2c906ca34275895e15883c59c78a" member_type: VOTER last_known_addr { host: "127.22.154.124" port: 43081 } }
02:06:50.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:50.772332 23679 leader_election.cc:288] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 35f4ec22b53d435fb9e5ce86ca372e26 (127.22.154.126:33485), 5a3719d34abf44e088edadebf21b430c (127.22.154.125:41045)
02:06:50.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:50.773859 23527 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.154.126:33485: connect: Connection refused (error 111) [suppressed 2 similar messages]
02:06:50.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:50.777704 23527 leader_election.cc:334] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 35f4ec22b53d435fb9e5ce86ca372e26 (127.22.154.126:33485): Network error: Client connection negotiation failed: client connection to 127.22.154.126:33485: connect: Connection refused (error 111)
02:06:50.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:50.778215 23525 leader_election.cc:334] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 5a3719d34abf44e088edadebf21b430c (127.22.154.125:41045): Network error: Client connection negotiation failed: client connection to 127.22.154.125:41045: connect: Connection refused (error 111)
02:06:50.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:50.778563 23525 leader_election.cc:302] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: c39d2c906ca34275895e15883c59c78a; no voters: 35f4ec22b53d435fb9e5ce86ca372e26, 5a3719d34abf44e088edadebf21b430c
02:06:50.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:50.779206 23679 raft_consensus.cc:2726] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
02:06:51.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:51.677767 23592 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:06:51.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:51.678061 23592 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:06:52.228 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.154.126:33485(127.22.154.126:33485)] connecting to peer
02:06:52.229 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.154.125:41045(127.22.154.125:41045)] connecting to peer
02:06:52.231 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:06:52.231 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.154.125:41045(127.22.154.125:41045)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:06:52.231 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.154.125:41045: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:06:52.234 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:06:52.234 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.154.126:33485(127.22.154.126:33485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:06:52.235 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.154.126:33485: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:06:52.235 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045; will retry
02:06:52.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:52.681301 23592 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:06:52.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:52.681612 23592 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:06:53.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:53.207482 23687 raft_consensus.cc:482] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:06:53.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:53.207904 23687 raft_consensus.cc:504] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "35f4ec22b53d435fb9e5ce86ca372e26" member_type: VOTER last_known_addr { host: "127.22.154.126" port: 33485 } } peers { permanent_uuid: "5a3719d34abf44e088edadebf21b430c" member_type: VOTER last_known_addr { host: "127.22.154.125" port: 41045 } } peers { permanent_uuid: "c39d2c906ca34275895e15883c59c78a" member_type: VOTER last_known_addr { host: "127.22.154.124" port: 43081 } }
02:06:53.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:53.209730 23687 leader_election.cc:288] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 35f4ec22b53d435fb9e5ce86ca372e26 (127.22.154.126:33485), 5a3719d34abf44e088edadebf21b430c (127.22.154.125:41045)
02:06:53.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:53.215178 23527 leader_election.cc:334] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 35f4ec22b53d435fb9e5ce86ca372e26 (127.22.154.126:33485): Network error: Client connection negotiation failed: client connection to 127.22.154.126:33485: connect: Connection refused (error 111)
02:06:53.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:53.215641 23525 leader_election.cc:334] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 5a3719d34abf44e088edadebf21b430c (127.22.154.125:41045): Network error: Client connection negotiation failed: client connection to 127.22.154.125:41045: connect: Connection refused (error 111)
02:06:53.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:53.216034 23525 leader_election.cc:302] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: c39d2c906ca34275895e15883c59c78a; no voters: 35f4ec22b53d435fb9e5ce86ca372e26, 5a3719d34abf44e088edadebf21b430c
02:06:53.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:53.216701 23687 raft_consensus.cc:2726] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
02:06:53.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:53.684543 23592 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:06:53.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:53.684793 23592 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:06:54.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:54.688657 23592 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:06:54.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:54.688990 23592 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:06:55.249 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.154.126:33485(127.22.154.126:33485)] connecting to peer
02:06:55.250 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.154.125:41045(127.22.154.125:41045)] connecting to peer
02:06:55.251 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:06:55.251 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:06:55.252 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.154.125:41045(127.22.154.125:41045)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:06:55.252 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.154.126:33485(127.22.154.126:33485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:06:55.252 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.154.125:41045: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:06:55.252 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.154.126:33485: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:06:55.254 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045; will retry
02:06:55.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 02:06:55 dist-test-slave-nz2j krb5kdc[23158](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790388415, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.154.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:06:55.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:55.370019 23520 init.cc:295] Successfully reacquired a new kerberos TGT
02:06:55.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:55.691946 23592 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:06:55.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:55.692389 23592 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:06:55.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:55.692724 23506 master_runner.cc:186] Error getting master registration for 127.22.154.126:33485: Network error: Client connection negotiation failed: client connection to 127.22.154.126:33485: connect: Connection refused (error 111),
02:06:55.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:55.693066 23506 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:06:56.268 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.154.126:33485(127.22.154.126:33485)] connecting to peer
02:06:56.270 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.154.125:41045(127.22.154.125:41045)] connecting to peer
02:06:56.271 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:06:56.272 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.154.126:33485(127.22.154.126:33485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:06:56.272 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.154.126:33485: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:06:56.273 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:06:56.274 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.154.125:41045(127.22.154.125:41045)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:06:56.274 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.154.125:41045: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:06:56.274 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045; will retry
02:06:56.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:56.695250 23592 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:06:56.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:56.695513 23592 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:06:57.288 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.154.126:33485(127.22.154.126:33485)] connecting to peer
02:06:57.290 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.154.125:41045(127.22.154.125:41045)] connecting to peer
02:06:57.292 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:06:57.292 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.154.125:41045(127.22.154.125:41045)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:06:57.293 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.154.125:41045: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:06:57.294 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:06:57.294 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.154.126:33485(127.22.154.126:33485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:06:57.294 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.154.126:33485: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:06:57.296 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045; will retry
02:06:57.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:57.698539 23592 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:06:57.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:57.698781 23592 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:06:58.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:58.701552 23592 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:06:58.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:58.701793 23592 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:06:59.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:59.704552 23592 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:06:59.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:59.704797 23592 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:06:59.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:59.719269 23700 raft_consensus.cc:482] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:06:59.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:59.719691 23700 raft_consensus.cc:504] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "35f4ec22b53d435fb9e5ce86ca372e26" member_type: VOTER last_known_addr { host: "127.22.154.126" port: 33485 } } peers { permanent_uuid: "5a3719d34abf44e088edadebf21b430c" member_type: VOTER last_known_addr { host: "127.22.154.125" port: 41045 } } peers { permanent_uuid: "c39d2c906ca34275895e15883c59c78a" member_type: VOTER last_known_addr { host: "127.22.154.124" port: 43081 } }
02:06:59.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:59.721351 23700 leader_election.cc:288] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 35f4ec22b53d435fb9e5ce86ca372e26 (127.22.154.126:33485), 5a3719d34abf44e088edadebf21b430c (127.22.154.125:41045)
02:06:59.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:59.723831 23525 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.154.125:41045: connect: Connection refused (error 111) [suppressed 3 similar messages]
02:06:59.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:59.726410 23527 leader_election.cc:334] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 35f4ec22b53d435fb9e5ce86ca372e26 (127.22.154.126:33485): Network error: Client connection negotiation failed: client connection to 127.22.154.126:33485: connect: Connection refused (error 111)
02:06:59.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:06:59.726814 23525 leader_election.cc:334] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 5a3719d34abf44e088edadebf21b430c (127.22.154.125:41045): Network error: Client connection negotiation failed: client connection to 127.22.154.125:41045: connect: Connection refused (error 111)
02:06:59.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:59.727133 23525 leader_election.cc:302] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: c39d2c906ca34275895e15883c59c78a; no voters: 35f4ec22b53d435fb9e5ce86ca372e26, 5a3719d34abf44e088edadebf21b430c
02:06:59.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:06:59.727784 23700 raft_consensus.cc:2726] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
02:07:00.228 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.154.126:33485(127.22.154.126:33485)] connecting to peer
02:07:00.229 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.154.125:41045(127.22.154.125:41045)] connecting to peer
02:07:00.231 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:07:00.231 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:07:00.231 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.154.125:41045(127.22.154.125:41045)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:07:00.231 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.154.126:33485(127.22.154.126:33485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:07:00.231 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.154.126:33485: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:07:00.231 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.154.125:41045: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:07:00.233 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045; will retry
02:07:00.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:07:00.695653 23506 master_runner.cc:186] Error getting master registration for 127.22.154.126:33485: Network error: Client connection negotiation failed: client connection to 127.22.154.126:33485: connect: Connection refused (error 111),
02:07:00.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:07:00.695976 23506 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:07:00.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:07:00.708005 23592 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:07:00.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:07:00.708369 23592 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:07:01.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:07:01.711254 23592 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:07:01.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:07:01.711494 23592 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:07:02.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:07:02.714238 23592 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:07:02.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:07:02.714476 23592 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:07:03.028 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.154.126:33485(127.22.154.126:33485)] connecting to peer
02:07:03.029 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.154.125:41045(127.22.154.125:41045)] connecting to peer
02:07:03.031 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:07:03.031 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:07:03.031 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.154.125:41045(127.22.154.125:41045)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:07:03.032 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.154.126:33485(127.22.154.126:33485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:07:03.032 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.154.125:41045: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:07:03.032 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.154.126:33485: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:07:03.033 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045; will retry
02:07:03.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:07:03.717295 23592 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:07:03.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:07:03.717533 23592 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:07:04.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:07:04.721424 23592 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:07:04.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:07:04.721740 23592 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:07:04.968 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.154.126:33485(127.22.154.126:33485)] connecting to peer
02:07:04.970 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.154.125:41045(127.22.154.125:41045)] connecting to peer
02:07:04.971 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:07:04.971 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:07:04.972 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.154.126:33485(127.22.154.126:33485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:07:04.972 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.154.125:41045(127.22.154.125:41045)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:07:04.972 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.154.126:33485: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:07:04.972 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.154.125:41045: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:07:04.974 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045; will retry
02:07:05.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 02:07:05 dist-test-slave-nz2j krb5kdc[23158](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790388425, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.154.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:07:05.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:07:05.373119 23520 init.cc:295] Successfully reacquired a new kerberos TGT
02:07:05.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:07:05.699702 23506 master_runner.cc:186] Error getting master registration for 127.22.154.126:33485: Network error: Client connection negotiation failed: client connection to 127.22.154.126:33485: connect: Connection refused (error 111),
02:07:05.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:07:05.700095 23506 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:07:05.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:07:05.725072 23592 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:07:05.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:07:05.725359 23592 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:07:06.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:07:06.728288 23592 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:07:06.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:07:06.728619 23592 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:07:06.968 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.154.126:33485(127.22.154.126:33485)] connecting to peer
02:07:06.969 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.154.125:41045(127.22.154.125:41045)] connecting to peer
02:07:06.970 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:07:06.971 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.154.126:33485(127.22.154.126:33485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:07:06.971 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.154.126:33485: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:07:06.972 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:07:06.972 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.154.125:41045(127.22.154.125:41045)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:07:06.972 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.154.125:41045: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:07:06.973 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045; will retry
02:07:07.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:07:07.731460 23592 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:07:07.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:07:07.731701 23592 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:07:08.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:07:08.734513 23592 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:07:08.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:07:08.734754 23592 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:07:09.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:07:09.737655 23592 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:07:09.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:07:09.737907 23592 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:07:09.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:07:09.981935 23715 raft_consensus.cc:482] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:07:09.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:07:09.982407 23715 raft_consensus.cc:504] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "35f4ec22b53d435fb9e5ce86ca372e26" member_type: VOTER last_known_addr { host: "127.22.154.126" port: 33485 } } peers { permanent_uuid: "5a3719d34abf44e088edadebf21b430c" member_type: VOTER last_known_addr { host: "127.22.154.125" port: 41045 } } peers { permanent_uuid: "c39d2c906ca34275895e15883c59c78a" member_type: VOTER last_known_addr { host: "127.22.154.124" port: 43081 } }
02:07:09.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:07:09.983947 23715 leader_election.cc:288] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 35f4ec22b53d435fb9e5ce86ca372e26 (127.22.154.126:33485), 5a3719d34abf44e088edadebf21b430c (127.22.154.125:41045)
02:07:09.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:07:09.986234 23525 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.154.125:41045: connect: Connection refused (error 111) [suppressed 1 similar messages]
02:07:09.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:07:09.988981 23527 leader_election.cc:334] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 35f4ec22b53d435fb9e5ce86ca372e26 (127.22.154.126:33485): Network error: Client connection negotiation failed: client connection to 127.22.154.126:33485: connect: Connection refused (error 111)
02:07:09.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:07:09.989490 23525 leader_election.cc:334] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 5a3719d34abf44e088edadebf21b430c (127.22.154.125:41045): Network error: Client connection negotiation failed: client connection to 127.22.154.125:41045: connect: Connection refused (error 111)
02:07:09.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:07:09.989912 23525 leader_election.cc:302] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: c39d2c906ca34275895e15883c59c78a; no voters: 35f4ec22b53d435fb9e5ce86ca372e26, 5a3719d34abf44e088edadebf21b430c
02:07:09.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:07:09.990535 23715 raft_consensus.cc:2726] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
02:07:10.528 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.154.126:33485(127.22.154.126:33485)] connecting to peer
02:07:10.529 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.154.125:41045(127.22.154.125:41045)] connecting to peer
02:07:10.530 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:07:10.530 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:07:10.530 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.154.126:33485(127.22.154.126:33485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:07:10.530 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.154.125:41045(127.22.154.125:41045)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:07:10.530 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.154.126:33485: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:07:10.530 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.154.125:41045: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:07:10.532 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045; will retry
02:07:10.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:07:10.702536 23506 master_runner.cc:186] Error getting master registration for 127.22.154.126:33485: Network error: Client connection negotiation failed: client connection to 127.22.154.126:33485: connect: Connection refused (error 111),
02:07:10.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:07:10.702900 23506 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:07:10.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:07:10.740810 23592 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:07:10.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:07:10.741072 23592 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:07:11.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:07:11.743860 23592 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:07:11.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:07:11.744113 23592 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:07:12.248 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.154.126:33485(127.22.154.126:33485)] connecting to peer
02:07:12.249 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.154.125:41045(127.22.154.125:41045)] connecting to peer
02:07:12.250 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:07:12.250 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:07:12.250 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.154.126:33485(127.22.154.126:33485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:07:12.250 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.154.125:41045(127.22.154.125:41045)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:07:12.250 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.154.125:41045: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:07:12.250 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.154.126:33485: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:07:12.253 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045; will retry
02:07:12.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:07:12.746901 23592 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:07:12.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:07:12.747313 23592 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:07:13.268 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.154.126:33485(127.22.154.126:33485)] connecting to peer
02:07:13.269 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.154.125:41045(127.22.154.125:41045)] connecting to peer
02:07:13.270 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:07:13.270 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:07:13.270 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.154.126:33485(127.22.154.126:33485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:07:13.270 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.154.125:41045(127.22.154.125:41045)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:07:13.271 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.154.125:41045: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:07:13.271 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.154.126:33485: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:07:13.272 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045; will retry
02:07:13.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:07:13.750258 23592 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:07:13.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:07:13.750511 23592 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:07:14.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:07:14.754163 23592 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:07:14.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:07:14.754554 23592 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:07:15.028 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.154.126:33485(127.22.154.126:33485)] connecting to peer
02:07:15.029 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.154.125:41045(127.22.154.125:41045)] connecting to peer
02:07:15.030 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:07:15.030 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:07:15.030 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.154.126:33485(127.22.154.126:33485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:07:15.030 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.154.125:41045(127.22.154.125:41045)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:07:15.031 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.154.126:33485: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:07:15.031 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.154.125:41045: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:07:15.033 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045; will retry
02:07:15.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 02:07:15 dist-test-slave-nz2j krb5kdc[23158](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790388435, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.154.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:07:15.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:07:15.375731 23520 init.cc:295] Successfully reacquired a new kerberos TGT
02:07:15.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:07:15.706048 23506 master_runner.cc:186] Error getting master registration for 127.22.154.126:33485: Network error: Client connection negotiation failed: client connection to 127.22.154.126:33485: connect: Connection refused (error 111),
02:07:15.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:07:15.706477 23506 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:07:15.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:07:15.757241 23592 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:07:15.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:07:15.757526 23592 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:07:16.668 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.154.126:33485(127.22.154.126:33485)] connecting to peer
02:07:16.669 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.154.125:41045(127.22.154.125:41045)] connecting to peer
02:07:16.669 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:07:16.670 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:07:16.670 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.154.126:33485(127.22.154.126:33485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:07:16.670 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.154.125:41045(127.22.154.125:41045)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:07:16.670 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.154.126:33485: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:07:16.670 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.154.125:41045: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:07:16.672 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045; will retry
02:07:16.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:07:16.759884 23592 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c39d2c906ca34275895e15883c59c78a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:07:16.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 02:07:16.760159 23592 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:07:17.127 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.154.126:33485(127.22.154.126:33485)] connecting to peer
02:07:17.128 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.154.125:41045(127.22.154.125:41045)] connecting to peer
02:07:17.129 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:07:17.129 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:07:17.129 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.154.125:41045(127.22.154.125:41045)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:07:17.130 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.154.126:33485(127.22.154.126:33485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:07:17.130 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.154.125:41045: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045
02:07:17.130 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.154.126:33485: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
02:07:17.132 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045; will retry
02:07:17.135 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=29586), Traces: [0ms] refreshing cache from master, [43ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.126:33485, [117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.124:43081, [118ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.125:41045, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.126:33485: Network error: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [186ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.125:41045: Network error: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [504ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.124:43081: OK, [508ms] delaying RPC due to: Service unavailable: Master config (127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [515ms] refreshing cache from master, [516ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.126:33485, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.124:43081, [520ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.125:41045, [523ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.125:41045: Network error: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [526ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.126:33485: Network error: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [528ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.124:43081: OK, [529ms] delaying RPC due to: Service unavailable: Master config (127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [535ms] refreshing cache from master, [535ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.126:33485, [536ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.124:43081, [536ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.125:41045, [542ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.125:41045: Network error: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [543ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.126:33485: Network error: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [545ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.124:43081: OK, [546ms] delaying RPC due to: Service unavailable: Master config (127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [555ms] refreshing cache from master, [555ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.126:33485, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.124:43081, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.125:41045, [560ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.125:41045: Network error: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [563ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.124:43081: OK, [563ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.126:33485: Network error: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [565ms] delaying RPC due to: Service unavailable: Master config (127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [576ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.126:33485, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.124:43081, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.125:41045, [580ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.125:41045: Network error: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [581ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.126:33485: Network error: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [583ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.124:43081: OK, [583ms] delaying RPC due to: Service unavailable: Master config (127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [615ms] refreshing cache from master, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.126:33485, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.124:43081, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.125:41045, [620ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.125:41045: Network error: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [620ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.126:33485: Network error: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [622ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.124:43081: OK, [623ms] delaying RPC due to: Service unavailable: Master config (127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [695ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.126:33485, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.124:43081, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.125:41045, [700ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.125:41045: Network error: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [701ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.124:43081: OK, [702ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.126:33485: Network error: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [703ms] delaying RPC due to: Service unavailable: Master config (127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [815ms] refreshing cache from master, [816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.126:33485, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.124:43081, [818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.125:41045, [820ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.126:33485: Network error: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [820ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.125:41045: Network error: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [822ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.124:43081: OK, [823ms] delaying RPC due to: Service unavailable: Master config (127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [955ms] refreshing cache from master, [956ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.126:33485, [957ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.124:43081, [957ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.125:41045, [960ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.125:41045: Network error: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [961ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.126:33485: Network error: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [963ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.124:43081: OK, [963ms] delaying RPC due to: Service unavailable: Master config (127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [1196ms] refreshing cache from master, [1196ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.126:33485, [1197ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.124:43081, [1198ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.125:41045, [1200ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.125:41045: Network error: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [1201ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.126:33485: Network error: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [1203ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.124:43081: OK, [1203ms] delaying RPC due to: Service unavailable: Master config (127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [1375ms] refreshing cache from master, [1375ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.126:33485, [1376ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.124:43081, [1377ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.125:41045, [1379ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.125:41045: Network error: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [1380ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.126:33485: Network error: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [1381ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.124:43081: OK, [1382ms] delaying RPC due to: Service unavailable: Master config (127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [2735ms] refreshing cache from master, [2735ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.126:33485, [2738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.124:43081, [2738ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.126:33485: Network error: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [2738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.125:41045, [2741ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.125:41045: Network error: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [2742ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.124:43081: OK, [2742ms] delaying RPC due to: Service unavailable: Master config (127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [4675ms] refreshing cache from master, [4676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.126:33485, [4677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.124:43081, [4677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.125:41045, [4679ms] too many traces: truncated at 100 traces, deferred=Deferred@638338120(state=PENDING, result=null, callback=(continuation of Deferred@1694622589 after retry RPC after error@808946204) -> (continuation of Deferred@55211295 after retry RPC after error@1193299770) -> (continuation of Deferred@768624296 after retry RPC after error@565923428) -> (continuation of Deferred@1904813505 after retry RPC after error@873856341) -> (continuation of Deferred@459217017 after retry RPC after error@1617734936) -> (continuation of Deferred@1483942905 after retry RPC after error@1125620471) -> (continuation of Deferred@552132156 after retry RPC after error@636072788) -> (continuation of Deferred@1686403315 after retry RPC after error@465278646) -> (continuation of Deferred@2110038723 after retry RPC after error@366992700) -> (continuation of Deferred@1390111892 after retry RPC after error@361913951) -> (continuation of Deferred@1301816468 after retry RPC after error@650454566) -> (continuation of Deferred@796086694 after retry RPC after error@672436913) -> (continuation of Deferred@606384035 after retry RPC after error@736057623) -> (continuation of Deferred@513069995 after retry RPC after error@312036306) -> (continuation of Deferred@323073941 after retry RPC after error@1107772171) -> (continuation of Deferred@2058799467 after retry RPC after error@573164084) -> (continuation of Deferred@1019886843 after retry RPC after error@1152200562) -> (continuation of Deferred@136076061 after retry RPC after error@1686657088) -> (continuation of Deferred@155753934 after retry RPC after error@607380778) -> (continuation of Deferred@1318950402 after retry RPC after error@1760660937) -> (continuation of Deferred@630901748 after retry RPC after error@507211992) -> (continuation of Deferred@1018484891 after retry RPC after error@1489999105) -> (continuation of Deferred@729362567 after retry RPC after error@1845573760) -> (continuation of Deferred@1953191387 after retry RPC after error@1344844234) -> (continuation of Deferred@842489408 after retry RPC after error@1427482046), errback=(continuation of Deferred@1694622589 after retry RPC after error@808946204) -> (continuation of Deferred@55211295 after retry RPC after error@1193299770) -> (continuation of Deferred@768624296 after retry RPC after error@565923428) -> (continuation of Deferred@1904813505 after retry RPC after error@873856341) -> (continuation of Deferred@459217017 after retry RPC after error@1617734936) -> (continuation of Deferred@1483942905 after retry RPC after error@1125620471) -> (continuation of Deferred@552132156 after retry RPC after error@636072788) -> (continuation of Deferred@1686403315 after retry RPC after error@465278646) -> (continuation of Deferred@2110038723 after retry RPC after error@366992700) -> (continuation of Deferred@1390111892 after retry RPC after error@361913951) -> (continuation of Deferred@1301816468 after retry RPC after error@650454566) -> (continuation of Deferred@796086694 after retry RPC after error@672436913) -> (continuation of Deferred@606384035 after retry RPC after error@736057623) -> (continuation of Deferred@513069995 after retry RPC after error@312036306) -> (continuation of Deferred@323073941 after retry RPC after error@1107772171) -> (continuation of Deferred@2058799467 after retry RPC after error@573164084) -> (continuation of Deferred@1019886843 after retry RPC after error@1152200562) -> (continuation of Deferred@136076061 after retry RPC after error@1686657088) -> (continuation of Deferred@155753934 after retry RPC after error@607380778) -> (continuation of Deferred@1318950402 after retry RPC after error@1760660937) -> (continuation of Deferred@630901748 after retry RPC after error@507211992) -> (continuation of Deferred@1018484891 after retry RPC after error@1489999105) -> (continuation of Deferred@729362567 after retry RPC after error@1845573760) -> (continuation of Deferred@1953191387 after retry RPC after error@1344844234) -> (continuation of Deferred@842489408 after retry RPC after error@1427482046)))
02:07:17.141 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.22.154.124:43081(127.22.154.124:43081)] disconnecting while in state READY
02:07:17.142 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.22.154.125:41045(127.22.154.125:41045)] disconnecting while in state TERMINATED
02:07:17.142 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.22.154.126:33485(127.22.154.126:33485)] disconnecting while in state TERMINATED
02:07:17.146 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
02:07:17.148 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.22.154.124:43081(127.22.154.124:43081)] handling channelInactive
02:07:17.148 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
02:07:17.148 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.154.124:43081(127.22.154.124:43081)] cleaning up while in state READY due to: connection closed
02:07:17.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:07:17.152729 23145 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskQ73J9W/build/tsan/bin/kudu with pid 23506
02:07:18.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-26T02:07:18Z chronyd exiting
02:07:19.349 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
02:07:19.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:07:19.350716 22419 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskQ73J9W/build/tsan/bin/kudu with pid 22699
02:07:19.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:07:19.376174 22419 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskQ73J9W/build/tsan/bin/kudu with pid 22859
02:07:19.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:07:19.403612 22419 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskQ73J9W/build/tsan/bin/kudu with pid 23004
02:07:19.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:07:19.428220 22419 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskQ73J9W/build/tsan/bin/kudu with pid 22431
02:07:19.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:07:19.458815 22419 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskQ73J9W/build/tsan/bin/kudu with pid 22501
02:07:19.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 02:07:19.487360 22419 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskQ73J9W/build/tsan/bin/kudu with pid 22576
02:07:20.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-26T02:07:20Z chronyd exiting
02:07:21.678 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=29586), Traces: [0ms] refreshing cache from master, [43ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.126:33485, [117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.124:43081, [118ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.125:41045, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.126:33485: Network error: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [186ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.125:41045: Network error: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [504ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.124:43081: OK, [508ms] delaying RPC due to: Service unavailable: Master config (127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [515ms] refreshing cache from master, [516ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.126:33485, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.124:43081, [520ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.125:41045, [523ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.125:41045: Network error: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [526ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.126:33485: Network error: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [528ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.124:43081: OK, [529ms] delaying RPC due to: Service unavailable: Master config (127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [535ms] refreshing cache from master, [535ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.126:33485, [536ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.124:43081, [536ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.125:41045, [542ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.125:41045: Network error: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [543ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.126:33485: Network error: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [545ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.124:43081: OK, [546ms] delaying RPC due to: Service unavailable: Master config (127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [555ms] refreshing cache from master, [555ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.126:33485, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.124:43081, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.125:41045, [560ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.125:41045: Network error: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [563ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.124:43081: OK, [563ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.126:33485: Network error: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [565ms] delaying RPC due to: Service unavailable: Master config (127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [576ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.126:33485, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.124:43081, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.125:41045, [580ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.125:41045: Network error: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [581ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.126:33485: Network error: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [583ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.124:43081: OK, [583ms] delaying RPC due to: Service unavailable: Master config (127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [615ms] refreshing cache from master, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.126:33485, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.124:43081, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.125:41045, [620ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.125:41045: Network error: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [620ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.126:33485: Network error: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [622ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.124:43081: OK, [623ms] delaying RPC due to: Service unavailable: Master config (127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [695ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.126:33485, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.124:43081, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.125:41045, [700ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.125:41045: Network error: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [701ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.124:43081: OK, [702ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.126:33485: Network error: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [703ms] delaying RPC due to: Service unavailable: Master config (127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [815ms] refreshing cache from master, [816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.126:33485, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.124:43081, [818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.125:41045, [820ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.126:33485: Network error: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [820ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.125:41045: Network error: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [822ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.124:43081: OK, [823ms] delaying RPC due to: Service unavailable: Master config (127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [955ms] refreshing cache from master, [956ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.126:33485, [957ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.124:43081, [957ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.125:41045, [960ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.125:41045: Network error: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [961ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.126:33485: Network error: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [963ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.124:43081: OK, [963ms] delaying RPC due to: Service unavailable: Master config (127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [1196ms] refreshing cache from master, [1196ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.126:33485, [1197ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.124:43081, [1198ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.125:41045, [1200ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.125:41045: Network error: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [1201ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.126:33485: Network error: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [1203ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.124:43081: OK, [1203ms] delaying RPC due to: Service unavailable: Master config (127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [1375ms] refreshing cache from master, [1375ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.126:33485, [1376ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.124:43081, [1377ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.125:41045, [1379ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.125:41045: Network error: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [1380ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.126:33485: Network error: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [1381ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.124:43081: OK, [1382ms] delaying RPC due to: Service unavailable: Master config (127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [2735ms] refreshing cache from master, [2735ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.126:33485, [2738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.124:43081, [2738ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.126:33485: Network error: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [2738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.125:41045, [2741ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.125:41045: Network error: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [2742ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.124:43081: OK, [2742ms] delaying RPC due to: Service unavailable: Master config (127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [4675ms] refreshing cache from master, [4676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.126:33485, [4677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.124:43081, [4677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.125:41045, [4679ms] too many traces: truncated at 100 traces, deferred=Deferred@638338120(state=PENDING, result=null, callback=(continuation of Deferred@1694622589 after retry RPC after error@808946204) -> (continuation of Deferred@55211295 after retry RPC after error@1193299770) -> (continuation of Deferred@768624296 after retry RPC after error@565923428) -> (continuation of Deferred@1904813505 after retry RPC after error@873856341) -> (continuation of Deferred@459217017 after retry RPC after error@1617734936) -> (continuation of Deferred@1483942905 after retry RPC after error@1125620471) -> (continuation of Deferred@552132156 after retry RPC after error@636072788) -> (continuation of Deferred@1686403315 after retry RPC after error@465278646) -> (continuation of Deferred@2110038723 after retry RPC after error@366992700) -> (continuation of Deferred@1390111892 after retry RPC after error@361913951) -> (continuation of Deferred@1301816468 after retry RPC after error@650454566) -> (continuation of Deferred@796086694 after retry RPC after error@672436913) -> (continuation of Deferred@606384035 after retry RPC after error@736057623) -> (continuation of Deferred@513069995 after retry RPC after error@312036306) -> (continuation of Deferred@323073941 after retry RPC after error@1107772171) -> (continuation of Deferred@2058799467 after retry RPC after error@573164084) -> (continuation of Deferred@1019886843 after retry RPC after error@1152200562) -> (continuation of Deferred@136076061 after retry RPC after error@1686657088) -> (continuation of Deferred@155753934 after retry RPC after error@607380778) -> (continuation of Deferred@1318950402 after retry RPC after error@1760660937) -> (continuation of Deferred@630901748 after retry RPC after error@507211992) -> (continuation of Deferred@1018484891 after retry RPC after error@1489999105) -> (continuation of Deferred@729362567 after retry RPC after error@1845573760) -> (continuation of Deferred@1953191387 after retry RPC after error@1344844234) -> (continuation of Deferred@842489408 after retry RPC after error@1427482046), errback=(continuation of Deferred@1694622589 after retry RPC after error@808946204) -> (continuation of Deferred@55211295 after retry RPC after error@1193299770) -> (continuation of Deferred@768624296 after retry RPC after error@565923428) -> (continuation of Deferred@1904813505 after retry RPC after error@873856341) -> (continuation of Deferred@459217017 after retry RPC after error@1617734936) -> (continuation of Deferred@1483942905 after retry RPC after error@1125620471) -> (continuation of Deferred@552132156 after retry RPC after error@636072788) -> (continuation of Deferred@1686403315 after retry RPC after error@465278646) -> (continuation of Deferred@2110038723 after retry RPC after error@366992700) -> (continuation of Deferred@1390111892 after retry RPC after error@361913951) -> (continuation of Deferred@1301816468 after retry RPC after error@650454566) -> (continuation of Deferred@796086694 after retry RPC after error@672436913) -> (continuation of Deferred@606384035 after retry RPC after error@736057623) -> (continuation of Deferred@513069995 after retry RPC after error@312036306) -> (continuation of Deferred@323073941 after retry RPC after error@1107772171) -> (continuation of Deferred@2058799467 after retry RPC after error@573164084) -> (continuation of Deferred@1019886843 after retry RPC after error@1152200562) -> (continuation of Deferred@136076061 after retry RPC after error@1686657088) -> (continuation of Deferred@155753934 after retry RPC after error@607380778) -> (continuation of Deferred@1318950402 after retry RPC after error@1760660937) -> (continuation of Deferred@630901748 after retry RPC after error@507211992) -> (continuation of Deferred@1018484891 after retry RPC after error@1489999105) -> (continuation of Deferred@729362567 after retry RPC after error@1845573760) -> (continuation of Deferred@1953191387 after retry RPC after error@1344844234) -> (continuation of Deferred@842489408 after retry RPC after error@1427482046)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.22.154.126:33485
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=29586), Traces: [0ms] refreshing cache from master, [43ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.126:33485, [117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.124:43081, [118ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.125:41045, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.126:33485: Network error: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [186ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.125:41045: Network error: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [504ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.124:43081: OK, [508ms] delaying RPC due to: Service unavailable: Master config (127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [515ms] refreshing cache from master, [516ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.126:33485, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.124:43081, [520ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.125:41045, [523ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.125:41045: Network error: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [526ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.126:33485: Network error: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [528ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.124:43081: OK, [529ms] delaying RPC due to: Service unavailable: Master config (127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [535ms] refreshing cache from master, [535ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.126:33485, [536ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.124:43081, [536ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.125:41045, [542ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.125:41045: Network error: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [543ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.126:33485: Network error: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [545ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.124:43081: OK, [546ms] delaying RPC due to: Service unavailable: Master config (127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [555ms] refreshing cache from master, [555ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.126:33485, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.124:43081, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.125:41045, [560ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.125:41045: Network error: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [563ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.124:43081: OK, [563ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.126:33485: Network error: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [565ms] delaying RPC due to: Service unavailable: Master config (127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [576ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.126:33485, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.124:43081, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.125:41045, [580ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.125:41045: Network error: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [581ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.126:33485: Network error: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [583ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.124:43081: OK, [583ms] delaying RPC due to: Service unavailable: Master config (127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [615ms] refreshing cache from master, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.126:33485, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.124:43081, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.125:41045, [620ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.125:41045: Network error: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [620ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.126:33485: Network error: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [622ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.124:43081: OK, [623ms] delaying RPC due to: Service unavailable: Master config (127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [695ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.126:33485, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.124:43081, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.125:41045, [700ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.125:41045: Network error: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [701ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.124:43081: OK, [702ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.126:33485: Network error: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [703ms] delaying RPC due to: Service unavailable: Master config (127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [815ms] refreshing cache from master, [816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.126:33485, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.124:43081, [818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.125:41045, [820ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.126:33485: Network error: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [820ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.125:41045: Network error: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [822ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.124:43081: OK, [823ms] delaying RPC due to: Service unavailable: Master config (127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [955ms] refreshing cache from master, [956ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.126:33485, [957ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.124:43081, [957ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.125:41045, [960ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.125:41045: Network error: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [961ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.126:33485: Network error: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [963ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.124:43081: OK, [963ms] delaying RPC due to: Service unavailable: Master config (127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [1196ms] refreshing cache from master, [1196ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.126:33485, [1197ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.124:43081, [1198ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.125:41045, [1200ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.125:41045: Network error: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [1201ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.126:33485: Network error: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [1203ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.124:43081: OK, [1203ms] delaying RPC due to: Service unavailable: Master config (127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [1375ms] refreshing cache from master, [1375ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.126:33485, [1376ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.124:43081, [1377ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.125:41045, [1379ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.125:41045: Network error: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [1380ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.126:33485: Network error: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [1381ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.124:43081: OK, [1382ms] delaying RPC due to: Service unavailable: Master config (127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [2735ms] refreshing cache from master, [2735ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.126:33485, [2738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.124:43081, [2738ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.126:33485: Network error: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [2738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.125:41045, [2741ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.125:41045: Network error: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [2742ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.124:43081: OK, [2742ms] delaying RPC due to: Service unavailable: Master config (127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [4675ms] refreshing cache from master, [4676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.126:33485, [4677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.124:43081, [4677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.125:41045, [4679ms] too many traces: truncated at 100 traces, deferred=Deferred@638338120(state=PENDING, result=null, callback=(continuation of Deferred@1694622589 after retry RPC after error@808946204) -> (continuation of Deferred@55211295 after retry RPC after error@1193299770) -> (continuation of Deferred@768624296 after retry RPC after error@565923428) -> (continuation of Deferred@1904813505 after retry RPC after error@873856341) -> (continuation of Deferred@459217017 after retry RPC after error@1617734936) -> (continuation of Deferred@1483942905 after retry RPC after error@1125620471) -> (continuation of Deferred@552132156 after retry RPC after error@636072788) -> (continuation of Deferred@1686403315 after retry RPC after error@465278646) -> (continuation of Deferred@2110038723 after retry RPC after error@366992700) -> (continuation of Deferred@1390111892 after retry RPC after error@361913951) -> (continuation of Deferred@1301816468 after retry RPC after error@650454566) -> (continuation of Deferred@796086694 after retry RPC after error@672436913) -> (continuation of Deferred@606384035 after retry RPC after error@736057623) -> (continuation of Deferred@513069995 after retry RPC after error@312036306) -> (continuation of Deferred@323073941 after retry RPC after error@1107772171) -> (continuation of Deferred@2058799467 after retry RPC after error@573164084) -> (continuation of Deferred@1019886843 after retry RPC after error@1152200562) -> (continuation of Deferred@136076061 after retry RPC after error@1686657088) -> (continuation of Deferred@155753934 after retry RPC after error@607380778) -> (continuation of Deferred@1318950402 after retry RPC after error@1760660937) -> (continuation of Deferred@630901748 after retry RPC after error@507211992) -> (continuation of Deferred@1018484891 after retry RPC after error@1489999105) -> (continuation of Deferred@729362567 after retry RPC after error@1845573760) -> (continuation of Deferred@1953191387 after retry RPC after error@1344844234) -> (continuation of Deferred@842489408 after retry RPC after error@1427482046), errback=(continuation of Deferred@1694622589 after retry RPC after error@808946204) -> (continuation of Deferred@55211295 after retry RPC after error@1193299770) -> (continuation of Deferred@768624296 after retry RPC after error@565923428) -> (continuation of Deferred@1904813505 after retry RPC after error@873856341) -> (continuation of Deferred@459217017 after retry RPC after error@1617734936) -> (continuation of Deferred@1483942905 after retry RPC after error@1125620471) -> (continuation of Deferred@552132156 after retry RPC after error@636072788) -> (continuation of Deferred@1686403315 after retry RPC after error@465278646) -> (continuation of Deferred@2110038723 after retry RPC after error@366992700) -> (continuation of Deferred@1390111892 after retry RPC after error@361913951) -> (continuation of Deferred@1301816468 after retry RPC after error@650454566) -> (continuation of Deferred@796086694 after retry RPC after error@672436913) -> (continuation of Deferred@606384035 after retry RPC after error@736057623) -> (continuation of Deferred@513069995 after retry RPC after error@312036306) -> (continuation of Deferred@323073941 after retry RPC after error@1107772171) -> (continuation of Deferred@2058799467 after retry RPC after error@573164084) -> (continuation of Deferred@1019886843 after retry RPC after error@1152200562) -> (continuation of Deferred@136076061 after retry RPC after error@1686657088) -> (continuation of Deferred@155753934 after retry RPC after error@607380778) -> (continuation of Deferred@1318950402 after retry RPC after error@1760660937) -> (continuation of Deferred@630901748 after retry RPC after error@507211992) -> (continuation of Deferred@1018484891 after retry RPC after error@1489999105) -> (continuation of Deferred@729362567 after retry RPC after error@1845573760) -> (continuation of Deferred@1953191387 after retry RPC after error@1344844234) -> (continuation of Deferred@842489408 after retry RPC after error@1427482046)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.22.154.126:33485
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
02:07:22.991 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
02:07:22.991 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=29586), Traces: [0ms] refreshing cache from master, [43ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.126:33485, [117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.124:43081, [118ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.125:41045, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.126:33485: Network error: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [186ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.125:41045: Network error: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [504ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.124:43081: OK, [508ms] delaying RPC due to: Service unavailable: Master config (127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [515ms] refreshing cache from master, [516ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.126:33485, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.124:43081, [520ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.125:41045, [523ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.125:41045: Network error: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [526ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.126:33485: Network error: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [528ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.124:43081: OK, [529ms] delaying RPC due to: Service unavailable: Master config (127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [535ms] refreshing cache from master, [535ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.126:33485, [536ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.124:43081, [536ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.125:41045, [542ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.125:41045: Network error: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [543ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.126:33485: Network error: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [545ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.124:43081: OK, [546ms] delaying RPC due to: Service unavailable: Master config (127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [555ms] refreshing cache from master, [555ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.126:33485, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.124:43081, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.125:41045, [560ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.125:41045: Network error: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [563ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.124:43081: OK, [563ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.126:33485: Network error: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [565ms] delaying RPC due to: Service unavailable: Master config (127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [576ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.126:33485, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.124:43081, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.125:41045, [580ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.125:41045: Network error: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [581ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.126:33485: Network error: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [583ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.124:43081: OK, [583ms] delaying RPC due to: Service unavailable: Master config (127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [615ms] refreshing cache from master, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.126:33485, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.124:43081, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.125:41045, [620ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.125:41045: Network error: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [620ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.126:33485: Network error: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [622ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.124:43081: OK, [623ms] delaying RPC due to: Service unavailable: Master config (127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [695ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.126:33485, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.124:43081, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.125:41045, [700ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.125:41045: Network error: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [701ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.124:43081: OK, [702ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.126:33485: Network error: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [703ms] delaying RPC due to: Service unavailable: Master config (127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [815ms] refreshing cache from master, [816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.126:33485, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.124:43081, [818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.125:41045, [820ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.126:33485: Network error: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [820ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.125:41045: Network error: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [822ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.124:43081: OK, [823ms] delaying RPC due to: Service unavailable: Master config (127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [955ms] refreshing cache from master, [956ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.126:33485, [957ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.124:43081, [957ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.125:41045, [960ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.125:41045: Network error: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [961ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.126:33485: Network error: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [963ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.124:43081: OK, [963ms] delaying RPC due to: Service unavailable: Master config (127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [1196ms] refreshing cache from master, [1196ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.126:33485, [1197ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.124:43081, [1198ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.125:41045, [1200ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.125:41045: Network error: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [1201ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.126:33485: Network error: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [1203ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.124:43081: OK, [1203ms] delaying RPC due to: Service unavailable: Master config (127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [1375ms] refreshing cache from master, [1375ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.126:33485, [1376ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.124:43081, [1377ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.125:41045, [1379ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.125:41045: Network error: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [1380ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.126:33485: Network error: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [1381ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.124:43081: OK, [1382ms] delaying RPC due to: Service unavailable: Master config (127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [2735ms] refreshing cache from master, [2735ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.126:33485, [2738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.124:43081, [2738ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.126:33485: Network error: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [2738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.125:41045, [2741ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.125:41045: Network error: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [2742ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.124:43081: OK, [2742ms] delaying RPC due to: Service unavailable: Master config (127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [4675ms] refreshing cache from master, [4676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.126:33485, [4677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.124:43081, [4677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.125:41045, [4679ms] too many traces: truncated at 100 traces, deferred=Deferred@638338120(state=PENDING, result=null, callback=(continuation of Deferred@1694622589 after retry RPC after error@808946204) -> (continuation of Deferred@55211295 after retry RPC after error@1193299770) -> (continuation of Deferred@768624296 after retry RPC after error@565923428) -> (continuation of Deferred@1904813505 after retry RPC after error@873856341) -> (continuation of Deferred@459217017 after retry RPC after error@1617734936) -> (continuation of Deferred@1483942905 after retry RPC after error@1125620471) -> (continuation of Deferred@552132156 after retry RPC after error@636072788) -> (continuation of Deferred@1686403315 after retry RPC after error@465278646) -> (continuation of Deferred@2110038723 after retry RPC after error@366992700) -> (continuation of Deferred@1390111892 after retry RPC after error@361913951) -> (continuation of Deferred@1301816468 after retry RPC after error@650454566) -> (continuation of Deferred@796086694 after retry RPC after error@672436913) -> (continuation of Deferred@606384035 after retry RPC after error@736057623) -> (continuation of Deferred@513069995 after retry RPC after error@312036306) -> (continuation of Deferred@323073941 after retry RPC after error@1107772171) -> (continuation of Deferred@2058799467 after retry RPC after error@573164084) -> (continuation of Deferred@1019886843 after retry RPC after error@1152200562) -> (continuation of Deferred@136076061 after retry RPC after error@1686657088) -> (continuation of Deferred@155753934 after retry RPC after error@607380778) -> (continuation of Deferred@1318950402 after retry RPC after error@1760660937) -> (continuation of Deferred@630901748 after retry RPC after error@507211992) -> (continuation of Deferred@1018484891 after retry RPC after error@1489999105) -> (continuation of Deferred@729362567 after retry RPC after error@1845573760) -> (continuation of Deferred@1953191387 after retry RPC after error@1344844234) -> (continuation of Deferred@842489408 after retry RPC after error@1427482046), errback=(continuation of Deferred@1694622589 after retry RPC after error@808946204) -> (continuation of Deferred@55211295 after retry RPC after error@1193299770) -> (continuation of Deferred@768624296 after retry RPC after error@565923428) -> (continuation of Deferred@1904813505 after retry RPC after error@873856341) -> (continuation of Deferred@459217017 after retry RPC after error@1617734936) -> (continuation of Deferred@1483942905 after retry RPC after error@1125620471) -> (continuation of Deferred@552132156 after retry RPC after error@636072788) -> (continuation of Deferred@1686403315 after retry RPC after error@465278646) -> (continuation of Deferred@2110038723 after retry RPC after error@366992700) -> (continuation of Deferred@1390111892 after retry RPC after error@361913951) -> (continuation of Deferred@1301816468 after retry RPC after error@650454566) -> (continuation of Deferred@796086694 after retry RPC after error@672436913) -> (continuation of Deferred@606384035 after retry RPC after error@736057623) -> (continuation of Deferred@513069995 after retry RPC after error@312036306) -> (continuation of Deferred@323073941 after retry RPC after error@1107772171) -> (continuation of Deferred@2058799467 after retry RPC after error@573164084) -> (continuation of Deferred@1019886843 after retry RPC after error@1152200562) -> (continuation of Deferred@136076061 after retry RPC after error@1686657088) -> (continuation of Deferred@155753934 after retry RPC after error@607380778) -> (continuation of Deferred@1318950402 after retry RPC after error@1760660937) -> (continuation of Deferred@630901748 after retry RPC after error@507211992) -> (continuation of Deferred@1018484891 after retry RPC after error@1489999105) -> (continuation of Deferred@729362567 after retry RPC after error@1845573760) -> (continuation of Deferred@1953191387 after retry RPC after error@1344844234) -> (continuation of Deferred@842489408 after retry RPC after error@1427482046)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.22.154.126:33485
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=29586), Traces: [0ms] refreshing cache from master, [43ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.126:33485, [117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.124:43081, [118ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.125:41045, [182ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.126:33485: Network error: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [186ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.125:41045: Network error: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [504ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.124:43081: OK, [508ms] delaying RPC due to: Service unavailable: Master config (127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [515ms] refreshing cache from master, [516ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.126:33485, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.124:43081, [520ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.125:41045, [523ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.125:41045: Network error: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [526ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.126:33485: Network error: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [528ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.124:43081: OK, [529ms] delaying RPC due to: Service unavailable: Master config (127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [535ms] refreshing cache from master, [535ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.126:33485, [536ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.124:43081, [536ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.125:41045, [542ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.125:41045: Network error: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [543ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.126:33485: Network error: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [545ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.124:43081: OK, [546ms] delaying RPC due to: Service unavailable: Master config (127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [555ms] refreshing cache from master, [555ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.126:33485, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.124:43081, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.125:41045, [560ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.125:41045: Network error: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [563ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.124:43081: OK, [563ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.126:33485: Network error: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [565ms] delaying RPC due to: Service unavailable: Master config (127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [576ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.126:33485, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.124:43081, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.125:41045, [580ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.125:41045: Network error: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [581ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.126:33485: Network error: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [583ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.124:43081: OK, [583ms] delaying RPC due to: Service unavailable: Master config (127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [615ms] refreshing cache from master, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.126:33485, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.124:43081, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.125:41045, [620ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.125:41045: Network error: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [620ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.126:33485: Network error: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [622ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.124:43081: OK, [623ms] delaying RPC due to: Service unavailable: Master config (127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [695ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.126:33485, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.124:43081, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.125:41045, [700ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.125:41045: Network error: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [701ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.124:43081: OK, [702ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.126:33485: Network error: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [703ms] delaying RPC due to: Service unavailable: Master config (127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [815ms] refreshing cache from master, [816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.126:33485, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.124:43081, [818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.125:41045, [820ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.126:33485: Network error: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [820ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.125:41045: Network error: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [822ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.124:43081: OK, [823ms] delaying RPC due to: Service unavailable: Master config (127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [955ms] refreshing cache from master, [956ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.126:33485, [957ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.124:43081, [957ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.125:41045, [960ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.125:41045: Network error: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [961ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.126:33485: Network error: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [963ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.124:43081: OK, [963ms] delaying RPC due to: Service unavailable: Master config (127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [1196ms] refreshing cache from master, [1196ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.126:33485, [1197ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.124:43081, [1198ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.125:41045, [1200ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.125:41045: Network error: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [1201ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.126:33485: Network error: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [1203ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.124:43081: OK, [1203ms] delaying RPC due to: Service unavailable: Master config (127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [1375ms] refreshing cache from master, [1375ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.126:33485, [1376ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.124:43081, [1377ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.125:41045, [1379ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.125:41045: Network error: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [1380ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.126:33485: Network error: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [1381ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.124:43081: OK, [1382ms] delaying RPC due to: Service unavailable: Master config (127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [2735ms] refreshing cache from master, [2735ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.126:33485, [2738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.124:43081, [2738ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.126:33485: Network error: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485, [2738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.125:41045, [2741ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.125:41045: Network error: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [2742ms] Sub RPC ConnectToMaster: received response from server master-127.22.154.124:43081: OK, [2742ms] delaying RPC due to: Service unavailable: Master config (127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045, [4675ms] refreshing cache from master, [4676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.126:33485, [4677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.124:43081, [4677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.154.125:41045, [4679ms] too many traces: truncated at 100 traces, deferred=Deferred@638338120(state=PENDING, result=null, callback=(continuation of Deferred@1694622589 after retry RPC after error@808946204) -> (continuation of Deferred@55211295 after retry RPC after error@1193299770) -> (continuation of Deferred@768624296 after retry RPC after error@565923428) -> (continuation of Deferred@1904813505 after retry RPC after error@873856341) -> (continuation of Deferred@459217017 after retry RPC after error@1617734936) -> (continuation of Deferred@1483942905 after retry RPC after error@1125620471) -> (continuation of Deferred@552132156 after retry RPC after error@636072788) -> (continuation of Deferred@1686403315 after retry RPC after error@465278646) -> (continuation of Deferred@2110038723 after retry RPC after error@366992700) -> (continuation of Deferred@1390111892 after retry RPC after error@361913951) -> (continuation of Deferred@1301816468 after retry RPC after error@650454566) -> (continuation of Deferred@796086694 after retry RPC after error@672436913) -> (continuation of Deferred@606384035 after retry RPC after error@736057623) -> (continuation of Deferred@513069995 after retry RPC after error@312036306) -> (continuation of Deferred@323073941 after retry RPC after error@1107772171) -> (continuation of Deferred@2058799467 after retry RPC after error@573164084) -> (continuation of Deferred@1019886843 after retry RPC after error@1152200562) -> (continuation of Deferred@136076061 after retry RPC after error@1686657088) -> (continuation of Deferred@155753934 after retry RPC after error@607380778) -> (continuation of Deferred@1318950402 after retry RPC after error@1760660937) -> (continuation of Deferred@630901748 after retry RPC after error@507211992) -> (continuation of Deferred@1018484891 after retry RPC after error@1489999105) -> (continuation of Deferred@729362567 after retry RPC after error@1845573760) -> (continuation of Deferred@1953191387 after retry RPC after error@1344844234) -> (continuation of Deferred@842489408 after retry RPC after error@1427482046), errback=(continuation of Deferred@1694622589 after retry RPC after error@808946204) -> (continuation of Deferred@55211295 after retry RPC after error@1193299770) -> (continuation of Deferred@768624296 after retry RPC after error@565923428) -> (continuation of Deferred@1904813505 after retry RPC after error@873856341) -> (continuation of Deferred@459217017 after retry RPC after error@1617734936) -> (continuation of Deferred@1483942905 after retry RPC after error@1125620471) -> (continuation of Deferred@552132156 after retry RPC after error@636072788) -> (continuation of Deferred@1686403315 after retry RPC after error@465278646) -> (continuation of Deferred@2110038723 after retry RPC after error@366992700) -> (continuation of Deferred@1390111892 after retry RPC after error@361913951) -> (continuation of Deferred@1301816468 after retry RPC after error@650454566) -> (continuation of Deferred@796086694 after retry RPC after error@672436913) -> (continuation of Deferred@606384035 after retry RPC after error@736057623) -> (continuation of Deferred@513069995 after retry RPC after error@312036306) -> (continuation of Deferred@323073941 after retry RPC after error@1107772171) -> (continuation of Deferred@2058799467 after retry RPC after error@573164084) -> (continuation of Deferred@1019886843 after retry RPC after error@1152200562) -> (continuation of Deferred@136076061 after retry RPC after error@1686657088) -> (continuation of Deferred@155753934 after retry RPC after error@607380778) -> (continuation of Deferred@1318950402 after retry RPC after error@1760660937) -> (continuation of Deferred@630901748 after retry RPC after error@507211992) -> (continuation of Deferred@1018484891 after retry RPC after error@1489999105) -> (continuation of Deferred@729362567 after retry RPC after error@1845573760) -> (continuation of Deferred@1953191387 after retry RPC after error@1344844234) -> (continuation of Deferred@842489408 after retry RPC after error@1427482046)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.22.154.126:33485,127.22.154.124:43081,127.22.154.125:41045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.125:41045(127.22.154.125:41045): Connection refused: /127.22.154.125:41045,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.154.126:33485(127.22.154.126:33485): Connection refused: /127.22.154.126:33485
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.22.154.126:33485
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)