07:29:25.349 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
07:29:25.360 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task8eqvEo/test-tmp
07:29:25.363 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
07:29:25.366 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task8eqvEo/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
07:29:25.574 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
07:29:25.757 [DEBUG - main] (MiniKuduCluster.java:195) Response:
07:29:25.758 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
07:29:25.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:25.761152 8304 mini_oidc.cc:152] Starting JWKS server
07:29:25.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:25.765616 8304 webserver.cc:460] Webserver started at http://127.0.0.1:39571/ using document root <none> and password file <none>
07:29:25.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:25.766098 8304 mini_oidc.cc:189] Starting OIDC Discovery server
07:29:25.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:25.767710 8304 webserver.cc:460] Webserver started at http://127.0.0.1:43925/ using document root <none> and password file <none>
07:29:25.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-24T07:29:25Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
07:29:25.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-24T07:29:25Z Disabled control of system clock
07:29:25.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:25.816154 8304 external_mini_cluster.cc:1326] Running /tmp/dist-test-task8eqvEo/build/tsan/bin/kudu
07:29:25.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task8eqvEo/build/tsan/bin/kudu
07:29:25.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/master-0/wal
07:29:25.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/master-0/data
07:29:25.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:29:25.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:29:25.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:29:25.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:29:25.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:29:25.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:29:25.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/master-0/logs
07:29:25.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/master-0/data/info.pb
07:29:25.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:29:25.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:29:25.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:29:25.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:29:25.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:29:25.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:29:25.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:29:25.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:29:25.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:29:25.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:29:25.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:29:25.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:29:25.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.28.62:45427
07:29:25.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.28.62
07:29:25.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:29:25.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.28.20:40953
07:29:25.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:29:25.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:29:25.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:29:25.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:29:25.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.28.62:45427,127.8.28.61:34477,127.8.28.60:39483
07:29:25.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:29:25.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39571/jwks.json with env {}
07:29:26.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:26.128777 8316 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:29:26.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:26.129354 8316 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:29:26.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:26.130091 8316 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:29:26.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:26.162209 8316 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:29:26.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:26.162492 8316 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:29:26.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:26.162681 8316 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:29:26.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:26.162858 8316 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:29:26.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:26.163151 8316 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:29:26.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:26.163390 8316 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39571/jwks.json
07:29:26.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:26.199002 8316 master_runner.cc:386] Master server non-default flags:
07:29:26.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:29:26.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.28.20:40953
07:29:26.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:29:26.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:29:26.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/master-0/data
07:29:26.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/master-0/wal
07:29:26.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:29:26.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.28.62:45427,127.8.28.61:34477,127.8.28.60:39483
07:29:26.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:29:26.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:29:26.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:29:26.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.28.62:45427
07:29:26.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:29:26.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:29:26.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:29:26.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39571/jwks.json
07:29:26.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:29:26.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:29:26.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/master-0/data/info.pb
07:29:26.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.28.62
07:29:26.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:29:26.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:29:26.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:29:26.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:29:26.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:29:26.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:29:26.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/master-0/logs
07:29:26.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:29:26.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:29:26.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:29:26.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:29:26.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:29:26.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:29:26.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:29:26.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 24 Sep 2026 06:42:32 UTC on ccycloud.quasar-lgarko.root.comops.site
07:29:26.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4669406
07:29:26.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:29:26.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:26.200392 8316 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:29:26.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:26.202687 8316 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:29:26.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:26.222754 8322 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:29:27.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:26.222929 8323 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:29:27.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:26.225577 8316 server_base.cc:1034] running on GCE node
07:29:27.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:26.225817 8325 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:29:27.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:27.418308 8316 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:29:27.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:27.422130 8316 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:29:27.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:27.423588 8316 hybrid_clock.cc:648] HybridClock initialized: now 1790234967423556 us; error 50 us; skew 500 ppm
07:29:27.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:27.424399 8316 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:29:27.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:27.436184 8316 webserver.cc:460] Webserver started at http://127.8.28.62:38819/ using document root <none> and password file <none>
07:29:27.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:27.437186 8316 fs_manager.cc:359] Metadata directory not provided
07:29:27.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:27.437383 8316 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:29:27.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:27.437860 8316 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:29:27.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:27.442988 8316 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/master-0/data/instance:
07:29:27.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c25a52d16dc24d558d31e61573570268"
07:29:27.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:29:27 on dist-test-slave-hp5j"
07:29:27.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:27.444325 8316 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/master-0/wal/instance:
07:29:27.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c25a52d16dc24d558d31e61573570268"
07:29:27.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:29:27 on dist-test-slave-hp5j"
07:29:27.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:27.451987 8316 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.004s
07:29:27.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:27.458048 8332 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:29:27.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:27.459261 8316 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
07:29:27.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:27.459563 8316 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/master-0/data,/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/master-0/wal
07:29:27.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c25a52d16dc24d558d31e61573570268"
07:29:27.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:29:27 on dist-test-slave-hp5j"
07:29:27.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:27.459849 8316 fs_report.cc:383] FS layout report
07:29:27.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:29:27.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/master-0/wal
07:29:27.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/master-0/wal
07:29:27.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/master-0/data/data
07:29:27.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:29:27.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:29:27.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:29:27.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:29:27.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:29:27.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:29:27.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:29:27.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:29:27.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:29:27.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:29:27.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:29:27.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:27.497813 8316 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:29:27.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:27.499315 8316 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:29:27.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:27.499720 8316 kserver.cc:163] Server-wide thread pool size limit: 3276
07:29:27.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:27.580452 8316 rpc_server.cc:307] RPC server started. Bound to: 127.8.28.62:45427
07:29:27.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:27.580530 8383 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.8.28.62:45427 every 8 connection(s)
07:29:27.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:27.583452 8316 server_base.cc:1166] Dumped server information to /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/master-0/data/info.pb
07:29:27.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:27.589167 8384 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:29:27.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:27.591580 8304 external_mini_cluster.cc:1388] Started /tmp/dist-test-task8eqvEo/build/tsan/bin/kudu as pid 8316
07:29:27.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:27.591949 8304 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/master-0/wal/instance
07:29:27.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:27.595235 8304 external_mini_cluster.cc:1326] Running /tmp/dist-test-task8eqvEo/build/tsan/bin/kudu
07:29:27.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task8eqvEo/build/tsan/bin/kudu
07:29:27.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/master-1/wal
07:29:27.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/master-1/data
07:29:27.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:29:27.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:29:27.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:29:27.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:29:27.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:29:27.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:29:27.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/master-1/logs
07:29:27.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/master-1/data/info.pb
07:29:27.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:29:27.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:29:27.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:29:27.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:29:27.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:29:27.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:29:27.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:29:27.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:29:27.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:29:27.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:29:27.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:29:27.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:29:27.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.28.61:34477
07:29:27.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.28.61
07:29:27.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:29:27.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.28.20:40953
07:29:27.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:29:27.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:29:27.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:29:27.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:29:27.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.28.62:45427,127.8.28.61:34477,127.8.28.60:39483
07:29:27.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:29:27.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39571/jwks.json with env {}
07:29:27.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:27.605432 8384 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.28.62" port: 45427 } has no permanent_uuid. Determining permanent_uuid...
07:29:27.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:27.635259 8384 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.28.61" port: 34477 } has no permanent_uuid. Determining permanent_uuid...
07:29:27.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:27.637990 8335 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.28.61:34477: connect: Connection refused (error 111)
07:29:27.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:27.642460 8384 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.28.61:34477: Network error: Client connection negotiation failed: client connection to 127.8.28.61:34477: connect: Connection refused (error 111)
07:29:27.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:27.692024 8384 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.28.61" port: 34477 } attempt: 1
07:29:27.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:27.695947 8384 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.28.61:34477: Network error: Client connection negotiation failed: client connection to 127.8.28.61:34477: connect: Connection refused (error 111)
07:29:27.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:27.764469 8384 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.28.61" port: 34477 } attempt: 2
07:29:27.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:27.768270 8384 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.28.61:34477: Network error: Client connection negotiation failed: client connection to 127.8.28.61:34477: connect: Connection refused (error 111)
07:29:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:27.859675 8384 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.28.61" port: 34477 } attempt: 3
07:29:27.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:27.863337 8384 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.28.61:34477: Network error: Client connection negotiation failed: client connection to 127.8.28.61:34477: connect: Connection refused (error 111)
07:29:27.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:27.918715 8386 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:29:27.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:27.919252 8386 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:29:27.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:27.919852 8386 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:29:27.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:27.951184 8386 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:29:27.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:27.951478 8386 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:29:27.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:27.951674 8386 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:29:27.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:27.951855 8386 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:29:27.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:27.952116 8386 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:29:27.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:27.952343 8386 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39571/jwks.json
07:29:27.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:27.988338 8386 master_runner.cc:386] Master server non-default flags:
07:29:27.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:29:27.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.28.20:40953
07:29:27.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:29:27.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:29:27.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/master-1/data
07:29:27.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/master-1/wal
07:29:27.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:29:27.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.28.62:45427,127.8.28.61:34477,127.8.28.60:39483
07:29:27.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:29:27.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:29:27.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:29:27.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.28.61:34477
07:29:27.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:29:27.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:29:27.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:29:27.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39571/jwks.json
07:29:27.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:29:27.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:29:27.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/master-1/data/info.pb
07:29:27.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.28.61
07:29:27.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:29:27.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:29:27.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:29:27.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:29:27.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:29:27.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:29:27.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/master-1/logs
07:29:27.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:29:27.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:29:27.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:29:27.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:29:27.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:29:27.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:29:27.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:29:27.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 24 Sep 2026 06:42:32 UTC on ccycloud.quasar-lgarko.root.comops.site
07:29:27.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4669406
07:29:27.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:29:27.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:27.989573 8386 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:29:27.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:27.991554 8386 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:29:28.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:28.006711 8384 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.28.61" port: 34477 } attempt: 4
07:29:28.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:28.011576 8384 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.28.61:34477: Network error: Client connection negotiation failed: client connection to 127.8.28.61:34477: connect: Connection refused (error 111)
07:29:28.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:28.012766 8395 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:29:28.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:28.311123 8384 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.28.61" port: 34477 } attempt: 5
07:29:28.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:28.315155 8384 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.28.61:34477: Network error: Client connection negotiation failed: client connection to 127.8.28.61:34477: connect: Connection refused (error 111)
07:29:28.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:28.862835 8384 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.28.61" port: 34477 } attempt: 6
07:29:28.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:28.871666 8384 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.28.61:34477: Network error: Client connection negotiation failed: client connection to 127.8.28.61:34477: connect: Connection refused (error 111)
07:29:29.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:28.012773 8398 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:29:29.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:28.013980 8386 server_base.cc:1034] running on GCE node
07:29:29.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:28.013291 8396 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:29:29.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:29.228156 8386 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:29:29.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:29.231403 8386 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:29:29.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:29.234946 8386 hybrid_clock.cc:648] HybridClock initialized: now 1790234969234349 us; error 598 us; skew 500 ppm
07:29:29.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:29.236001 8386 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:29:29.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:29.242970 8386 webserver.cc:460] Webserver started at http://127.8.28.61:44553/ using document root <none> and password file <none>
07:29:29.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:29.244035 8386 fs_manager.cc:359] Metadata directory not provided
07:29:29.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:29.244313 8386 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:29:29.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:29.244845 8386 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:29:29.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:29.250044 8386 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/master-1/data/instance:
07:29:29.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ab6b10d629bc47bd902467fd80dc9d37"
07:29:29.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:29:29 on dist-test-slave-hp5j"
07:29:29.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:29.251289 8386 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/master-1/wal/instance:
07:29:29.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ab6b10d629bc47bd902467fd80dc9d37"
07:29:29.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:29:29 on dist-test-slave-hp5j"
07:29:29.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:29.258575 8386 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
07:29:29.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:29.264025 8407 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:29:29.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:29.265082 8386 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
07:29:29.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:29.265367 8386 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/master-1/data,/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/master-1/wal
07:29:29.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ab6b10d629bc47bd902467fd80dc9d37"
07:29:29.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:29:29 on dist-test-slave-hp5j"
07:29:29.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:29.265717 8386 fs_report.cc:383] FS layout report
07:29:29.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:29:29.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/master-1/wal
07:29:29.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/master-1/wal
07:29:29.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/master-1/data/data
07:29:29.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:29:29.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:29:29.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:29:29.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:29:29.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:29:29.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:29:29.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:29:29.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:29:29.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:29:29.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:29:29.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:29:29.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:29.306854 8386 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:29:29.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:29.308248 8386 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:29:29.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:29.308635 8386 kserver.cc:163] Server-wide thread pool size limit: 3276
07:29:29.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:29.384603 8386 rpc_server.cc:307] RPC server started. Bound to: 127.8.28.61:34477
07:29:29.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:29.384689 8458 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.8.28.61:34477 every 8 connection(s)
07:29:29.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:29.387358 8386 server_base.cc:1166] Dumped server information to /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/master-1/data/info.pb
07:29:29.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:29.393182 8459 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:29:29.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:29.394096 8304 external_mini_cluster.cc:1388] Started /tmp/dist-test-task8eqvEo/build/tsan/bin/kudu as pid 8386
07:29:29.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:29.394505 8304 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/master-1/wal/instance
07:29:29.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:29.396730 8304 external_mini_cluster.cc:1326] Running /tmp/dist-test-task8eqvEo/build/tsan/bin/kudu
07:29:29.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task8eqvEo/build/tsan/bin/kudu
07:29:29.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/master-2/wal
07:29:29.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/master-2/data
07:29:29.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:29:29.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:29:29.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:29:29.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:29:29.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:29:29.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:29:29.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/master-2/logs
07:29:29.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/master-2/data/info.pb
07:29:29.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:29:29.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:29:29.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:29:29.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:29:29.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:29:29.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:29:29.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:29:29.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:29:29.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:29:29.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:29:29.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:29:29.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:29:29.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.28.60:39483
07:29:29.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.28.60
07:29:29.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:29:29.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.28.20:40953
07:29:29.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:29:29.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:29:29.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:29:29.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:29:29.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.28.62:45427,127.8.28.61:34477,127.8.28.60:39483
07:29:29.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:29:29.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39571/jwks.json with env {}
07:29:29.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:29.409839 8459 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.28.62" port: 45427 } has no permanent_uuid. Determining permanent_uuid...
07:29:29.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:29.438233 8459 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.28.61" port: 34477 } has no permanent_uuid. Determining permanent_uuid...
07:29:29.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:29.453570 8459 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.28.60" port: 39483 } has no permanent_uuid. Determining permanent_uuid...
07:29:29.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:29.457048 8409 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.28.60:39483: connect: Connection refused (error 111)
07:29:29.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:29.462612 8459 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.28.60:39483: Network error: Client connection negotiation failed: client connection to 127.8.28.60:39483: connect: Connection refused (error 111)
07:29:29.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:29.512238 8459 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.28.60" port: 39483 } attempt: 1
07:29:29.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:29.515936 8459 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.28.60:39483: Network error: Client connection negotiation failed: client connection to 127.8.28.60:39483: connect: Connection refused (error 111)
07:29:29.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:29.584406 8459 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.28.60" port: 39483 } attempt: 2
07:29:29.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:29.588783 8459 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.28.60:39483: Network error: Client connection negotiation failed: client connection to 127.8.28.60:39483: connect: Connection refused (error 111)
07:29:29.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:29.680632 8459 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.28.60" port: 39483 } attempt: 3
07:29:29.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:29.684963 8459 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.28.60:39483: Network error: Client connection negotiation failed: client connection to 127.8.28.60:39483: connect: Connection refused (error 111)
07:29:29.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:29.726444 8461 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:29:29.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:29.726979 8461 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:29:29.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:29.727429 8461 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:29:29.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:29.757454 8461 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:29:29.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:29.757735 8461 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:29:29.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:29.757920 8461 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:29:29.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:29.758104 8461 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:29:29.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:29.758289 8461 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:29:29.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:29.758426 8461 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39571/jwks.json
07:29:29.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:29.793036 8461 master_runner.cc:386] Master server non-default flags:
07:29:29.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:29:29.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.28.20:40953
07:29:29.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:29:29.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:29:29.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/master-2/data
07:29:29.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/master-2/wal
07:29:29.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:29:29.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.28.62:45427,127.8.28.61:34477,127.8.28.60:39483
07:29:29.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:29:29.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:29:29.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:29:29.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.28.60:39483
07:29:29.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:29:29.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:29:29.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:29:29.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39571/jwks.json
07:29:29.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:29:29.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:29:29.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/master-2/data/info.pb
07:29:29.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.28.60
07:29:29.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:29:29.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:29:29.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:29:29.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:29:29.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:29:29.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:29:29.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/master-2/logs
07:29:29.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:29:29.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:29:29.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:29:29.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:29:29.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:29:29.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:29:29.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:29:29.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 24 Sep 2026 06:42:32 UTC on ccycloud.quasar-lgarko.root.comops.site
07:29:29.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4669406
07:29:29.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:29:29.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:29.794314 8461 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:29:29.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:29.796387 8461 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:29:29.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:29.817742 8471 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:29:29.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:29.828403 8459 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.28.60" port: 39483 } attempt: 4
07:29:29.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:29.832027 8459 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.28.60:39483: Network error: Client connection negotiation failed: client connection to 127.8.28.60:39483: connect: Connection refused (error 111)
07:29:29.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:29.932317 8384 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.28.61" port: 34477 } attempt: 7
07:29:29.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:29.951051 8384 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.28.60" port: 39483 } has no permanent_uuid. Determining permanent_uuid...
07:29:29.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:29.958696 8384 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.28.60:39483: Network error: Client connection negotiation failed: client connection to 127.8.28.60:39483: connect: Connection refused (error 111)
07:29:30.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:30.017289 8384 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.28.60" port: 39483 } attempt: 1
07:29:30.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:30.022553 8384 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.28.60:39483: Network error: Client connection negotiation failed: client connection to 127.8.28.60:39483: connect: Connection refused (error 111)
07:29:30.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:30.104151 8384 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.28.60" port: 39483 } attempt: 2
07:29:30.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:30.109141 8384 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.28.60:39483: Network error: Client connection negotiation failed: client connection to 127.8.28.60:39483: connect: Connection refused (error 111)
07:29:30.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:30.131426 8459 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.28.60" port: 39483 } attempt: 5
07:29:30.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:30.136977 8459 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.28.60:39483: Network error: Client connection negotiation failed: client connection to 127.8.28.60:39483: connect: Connection refused (error 111)
07:29:30.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:30.194607 8384 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.28.60" port: 39483 } attempt: 3
07:29:30.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:30.199446 8384 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.28.60:39483: Network error: Client connection negotiation failed: client connection to 127.8.28.60:39483: connect: Connection refused (error 111)
07:29:30.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:30.339962 8384 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.28.60" port: 39483 } attempt: 4
07:29:30.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:30.343457 8384 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.28.60:39483: Network error: Client connection negotiation failed: client connection to 127.8.28.60:39483: connect: Connection refused (error 111)
07:29:30.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:30.627122 8384 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.28.60" port: 39483 } attempt: 5
07:29:30.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:30.631513 8384 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.28.60:39483: Network error: Client connection negotiation failed: client connection to 127.8.28.60:39483: connect: Connection refused (error 111)
07:29:30.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:30.684527 8459 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.28.60" port: 39483 } attempt: 6
07:29:30.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:30.692516 8459 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.28.60:39483: Network error: Client connection negotiation failed: client connection to 127.8.28.60:39483: connect: Connection refused (error 111)
07:29:31.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:29.819640 8474 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:29:31.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:29.818099 8472 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:29:31.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:31.044989 8473 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
07:29:31.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.045027 8461 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
07:29:31.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.049350 8461 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:29:31.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.052393 8461 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:29:31.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.053821 8461 hybrid_clock.cc:648] HybridClock initialized: now 1790234971053779 us; error 61 us; skew 500 ppm
07:29:31.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.054617 8461 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:29:31.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.061311 8461 webserver.cc:460] Webserver started at http://127.8.28.60:44655/ using document root <none> and password file <none>
07:29:31.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.062260 8461 fs_manager.cc:359] Metadata directory not provided
07:29:31.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.062433 8461 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:29:31.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.062790 8461 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:29:31.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.070220 8461 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/master-2/data/instance:
07:29:31.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d9518af1d5b74701849ed730f86695b4"
07:29:31.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:29:31 on dist-test-slave-hp5j"
07:29:31.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.071832 8461 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/master-2/wal/instance:
07:29:31.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d9518af1d5b74701849ed730f86695b4"
07:29:31.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:29:31 on dist-test-slave-hp5j"
07:29:31.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.080778 8461 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.010s sys 0.001s
07:29:31.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.087309 8485 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:29:31.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.088514 8461 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.003s sys 0.003s
07:29:31.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.088799 8461 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/master-2/data,/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/master-2/wal
07:29:31.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d9518af1d5b74701849ed730f86695b4"
07:29:31.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:29:31 on dist-test-slave-hp5j"
07:29:31.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.089097 8461 fs_report.cc:383] FS layout report
07:29:31.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:29:31.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/master-2/wal
07:29:31.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/master-2/wal
07:29:31.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/master-2/data/data
07:29:31.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:29:31.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:29:31.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:29:31.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:29:31.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:29:31.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:29:31.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:29:31.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:29:31.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:29:31.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:29:31.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:29:31.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.127810 8461 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:29:31.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.129257 8461 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:29:31.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.129738 8461 kserver.cc:163] Server-wide thread pool size limit: 3276
07:29:31.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.184667 8384 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.28.60" port: 39483 } attempt: 6
07:29:31.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:31.192565 8384 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.28.60:39483: Network error: Client connection negotiation failed: client connection to 127.8.28.60:39483: connect: Connection refused (error 111)
07:29:31.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.213220 8461 rpc_server.cc:307] RPC server started. Bound to: 127.8.28.60:39483
07:29:31.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.213279 8538 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.8.28.60:39483 every 8 connection(s)
07:29:31.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.216964 8461 server_base.cc:1166] Dumped server information to /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/master-2/data/info.pb
07:29:31.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.218727 8304 external_mini_cluster.cc:1388] Started /tmp/dist-test-task8eqvEo/build/tsan/bin/kudu as pid 8461
07:29:31.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.219300 8304 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/master-2/wal/instance
07:29:31.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.224644 8539 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:29:31.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.245163 8539 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.28.62" port: 45427 } has no permanent_uuid. Determining permanent_uuid...
07:29:31.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.276314 8539 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.28.61" port: 34477 } has no permanent_uuid. Determining permanent_uuid...
07:29:31.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.289204 8539 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.28.60" port: 39483 } has no permanent_uuid. Determining permanent_uuid...
07:29:31.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.307782 8539 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d9518af1d5b74701849ed730f86695b4: Bootstrap starting.
07:29:31.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.314536 8539 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P d9518af1d5b74701849ed730f86695b4: Neither blocks nor log segments found. Creating new log.
07:29:31.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.316529 8539 log.cc:824] T 00000000000000000000000000000000 P d9518af1d5b74701849ed730f86695b4: Log is configured to *not* fsync() on all Append() calls
07:29:31.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.320636 8539 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d9518af1d5b74701849ed730f86695b4: No bootstrap required, opened a new log
07:29:31.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.343230 8539 raft_consensus.cc:348] T 00000000000000000000000000000000 P d9518af1d5b74701849ed730f86695b4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c25a52d16dc24d558d31e61573570268" member_type: VOTER last_known_addr { host: "127.8.28.62" port: 45427 } } peers { permanent_uuid: "ab6b10d629bc47bd902467fd80dc9d37" member_type: VOTER last_known_addr { host: "127.8.28.61" port: 34477 } } peers { permanent_uuid: "d9518af1d5b74701849ed730f86695b4" member_type: VOTER last_known_addr { host: "127.8.28.60" port: 39483 } }
07:29:31.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.343899 8539 raft_consensus.cc:374] T 00000000000000000000000000000000 P d9518af1d5b74701849ed730f86695b4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:29:31.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.344605 8539 raft_consensus.cc:729] T 00000000000000000000000000000000 P d9518af1d5b74701849ed730f86695b4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d9518af1d5b74701849ed730f86695b4, State: Initialized, Role: FOLLOWER
07:29:31.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.345602 8539 consensus_queue.cc:260] T 00000000000000000000000000000000 P d9518af1d5b74701849ed730f86695b4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c25a52d16dc24d558d31e61573570268" member_type: VOTER last_known_addr { host: "127.8.28.62" port: 45427 } } peers { permanent_uuid: "ab6b10d629bc47bd902467fd80dc9d37" member_type: VOTER last_known_addr { host: "127.8.28.61" port: 34477 } } peers { permanent_uuid: "d9518af1d5b74701849ed730f86695b4" member_type: VOTER last_known_addr { host: "127.8.28.60" port: 39483 } }
07:29:31.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.347977 8548 sys_catalog.cc:455] T 00000000000000000000000000000000 P d9518af1d5b74701849ed730f86695b4 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c25a52d16dc24d558d31e61573570268" member_type: VOTER last_known_addr { host: "127.8.28.62" port: 45427 } } peers { permanent_uuid: "ab6b10d629bc47bd902467fd80dc9d37" member_type: VOTER last_known_addr { host: "127.8.28.61" port: 34477 } } peers { permanent_uuid: "d9518af1d5b74701849ed730f86695b4" member_type: VOTER last_known_addr { host: "127.8.28.60" port: 39483 } } }
07:29:31.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.349318 8548 sys_catalog.cc:458] T 00000000000000000000000000000000 P d9518af1d5b74701849ed730f86695b4 [sys.catalog]: This master's current role is: FOLLOWER
07:29:31.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.350111 8539 sys_catalog.cc:564] T 00000000000000000000000000000000 P d9518af1d5b74701849ed730f86695b4 [sys.catalog]: configured and running, proceeding with master startup.
07:29:31.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:31.377892 8559 catalog_manager.cc:1558] T 00000000000000000000000000000000 P d9518af1d5b74701849ed730f86695b4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:29:31.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:31.379640 8559 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:29:31.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:31.384820 8491 tablet.cc:2369] T 00000000000000000000000000000000 P d9518af1d5b74701849ed730f86695b4: Can't schedule compaction. Clean time has not been advanced past its initial value.
07:29:31.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.372500 8461 master_runner.cc:186] Error getting master registration for 127.8.28.62:45427: OK, instance_id { permanent_uuid: "c25a52d16dc24d558d31e61573570268" instance_seqno: 1790234967556974 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
07:29:31.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.408736 8461 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:29:31.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.414443 8548 raft_consensus.cc:482] T 00000000000000000000000000000000 P d9518af1d5b74701849ed730f86695b4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:29:31.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.414980 8548 raft_consensus.cc:504] T 00000000000000000000000000000000 P d9518af1d5b74701849ed730f86695b4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c25a52d16dc24d558d31e61573570268" member_type: VOTER last_known_addr { host: "127.8.28.62" port: 45427 } } peers { permanent_uuid: "ab6b10d629bc47bd902467fd80dc9d37" member_type: VOTER last_known_addr { host: "127.8.28.61" port: 34477 } } peers { permanent_uuid: "d9518af1d5b74701849ed730f86695b4" member_type: VOTER last_known_addr { host: "127.8.28.60" port: 39483 } }
07:29:31.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.417575 8548 leader_election.cc:288] T 00000000000000000000000000000000 P d9518af1d5b74701849ed730f86695b4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c25a52d16dc24d558d31e61573570268 (127.8.28.62:45427), ab6b10d629bc47bd902467fd80dc9d37 (127.8.28.61:34477)
07:29:31.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.753130 8459 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.28.60" port: 39483 } attempt: 7
07:29:31.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.775848 8459 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ab6b10d629bc47bd902467fd80dc9d37: Bootstrap starting.
07:29:31.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.783303 8459 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P ab6b10d629bc47bd902467fd80dc9d37: Neither blocks nor log segments found. Creating new log.
07:29:31.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.785974 8459 log.cc:824] T 00000000000000000000000000000000 P ab6b10d629bc47bd902467fd80dc9d37: Log is configured to *not* fsync() on all Append() calls
07:29:31.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.791214 8459 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ab6b10d629bc47bd902467fd80dc9d37: No bootstrap required, opened a new log
07:29:31.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.822232 8459 raft_consensus.cc:348] T 00000000000000000000000000000000 P ab6b10d629bc47bd902467fd80dc9d37 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c25a52d16dc24d558d31e61573570268" member_type: VOTER last_known_addr { host: "127.8.28.62" port: 45427 } } peers { permanent_uuid: "ab6b10d629bc47bd902467fd80dc9d37" member_type: VOTER last_known_addr { host: "127.8.28.61" port: 34477 } } peers { permanent_uuid: "d9518af1d5b74701849ed730f86695b4" member_type: VOTER last_known_addr { host: "127.8.28.60" port: 39483 } }
07:29:31.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.823082 8459 raft_consensus.cc:374] T 00000000000000000000000000000000 P ab6b10d629bc47bd902467fd80dc9d37 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:29:31.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.823338 8459 raft_consensus.cc:729] T 00000000000000000000000000000000 P ab6b10d629bc47bd902467fd80dc9d37 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ab6b10d629bc47bd902467fd80dc9d37, State: Initialized, Role: FOLLOWER
07:29:31.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.824661 8459 consensus_queue.cc:260] T 00000000000000000000000000000000 P ab6b10d629bc47bd902467fd80dc9d37 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c25a52d16dc24d558d31e61573570268" member_type: VOTER last_known_addr { host: "127.8.28.62" port: 45427 } } peers { permanent_uuid: "ab6b10d629bc47bd902467fd80dc9d37" member_type: VOTER last_known_addr { host: "127.8.28.61" port: 34477 } } peers { permanent_uuid: "d9518af1d5b74701849ed730f86695b4" member_type: VOTER last_known_addr { host: "127.8.28.60" port: 39483 } }
07:29:31.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.827601 8563 sys_catalog.cc:455] T 00000000000000000000000000000000 P ab6b10d629bc47bd902467fd80dc9d37 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c25a52d16dc24d558d31e61573570268" member_type: VOTER last_known_addr { host: "127.8.28.62" port: 45427 } } peers { permanent_uuid: "ab6b10d629bc47bd902467fd80dc9d37" member_type: VOTER last_known_addr { host: "127.8.28.61" port: 34477 } } peers { permanent_uuid: "d9518af1d5b74701849ed730f86695b4" member_type: VOTER last_known_addr { host: "127.8.28.60" port: 39483 } } }
07:29:31.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.828590 8563 sys_catalog.cc:458] T 00000000000000000000000000000000 P ab6b10d629bc47bd902467fd80dc9d37 [sys.catalog]: This master's current role is: FOLLOWER
07:29:31.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.829528 8459 sys_catalog.cc:564] T 00000000000000000000000000000000 P ab6b10d629bc47bd902467fd80dc9d37 [sys.catalog]: configured and running, proceeding with master startup.
07:29:31.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.829737 8434 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d9518af1d5b74701849ed730f86695b4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ab6b10d629bc47bd902467fd80dc9d37" is_pre_election: true
07:29:31.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.830567 8434 raft_consensus.cc:2454] T 00000000000000000000000000000000 P ab6b10d629bc47bd902467fd80dc9d37 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d9518af1d5b74701849ed730f86695b4 in term 0.
07:29:31.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.832449 8488 leader_election.cc:302] T 00000000000000000000000000000000 P d9518af1d5b74701849ed730f86695b4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: ab6b10d629bc47bd902467fd80dc9d37, d9518af1d5b74701849ed730f86695b4; no voters:
07:29:31.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.833221 8548 raft_consensus.cc:2781] T 00000000000000000000000000000000 P d9518af1d5b74701849ed730f86695b4 [term 0 FOLLOWER]: Leader pre-election won for term 1
07:29:31.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.833576 8548 raft_consensus.cc:482] T 00000000000000000000000000000000 P d9518af1d5b74701849ed730f86695b4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
07:29:31.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.833878 8548 raft_consensus.cc:3037] T 00000000000000000000000000000000 P d9518af1d5b74701849ed730f86695b4 [term 0 FOLLOWER]: Advancing to term 1
07:29:31.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.840479 8548 raft_consensus.cc:504] T 00000000000000000000000000000000 P d9518af1d5b74701849ed730f86695b4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c25a52d16dc24d558d31e61573570268" member_type: VOTER last_known_addr { host: "127.8.28.62" port: 45427 } } peers { permanent_uuid: "ab6b10d629bc47bd902467fd80dc9d37" member_type: VOTER last_known_addr { host: "127.8.28.61" port: 34477 } } peers { permanent_uuid: "d9518af1d5b74701849ed730f86695b4" member_type: VOTER last_known_addr { host: "127.8.28.60" port: 39483 } }
07:29:31.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.843317 8548 leader_election.cc:288] T 00000000000000000000000000000000 P d9518af1d5b74701849ed730f86695b4 [CANDIDATE]: Term 1 election: Requested vote from peers c25a52d16dc24d558d31e61573570268 (127.8.28.62:45427), ab6b10d629bc47bd902467fd80dc9d37 (127.8.28.61:34477)
07:29:31.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.844326 8434 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d9518af1d5b74701849ed730f86695b4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ab6b10d629bc47bd902467fd80dc9d37"
07:29:31.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.844887 8434 raft_consensus.cc:3037] T 00000000000000000000000000000000 P ab6b10d629bc47bd902467fd80dc9d37 [term 0 FOLLOWER]: Advancing to term 1
07:29:31.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.851605 8434 raft_consensus.cc:2454] T 00000000000000000000000000000000 P ab6b10d629bc47bd902467fd80dc9d37 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d9518af1d5b74701849ed730f86695b4 in term 1.
07:29:31.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.852557 8488 leader_election.cc:302] T 00000000000000000000000000000000 P d9518af1d5b74701849ed730f86695b4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: ab6b10d629bc47bd902467fd80dc9d37, d9518af1d5b74701849ed730f86695b4; no voters:
07:29:31.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.853330 8548 raft_consensus.cc:2781] T 00000000000000000000000000000000 P d9518af1d5b74701849ed730f86695b4 [term 1 FOLLOWER]: Leader election won for term 1
07:29:31.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.854939 8548 raft_consensus.cc:686] T 00000000000000000000000000000000 P d9518af1d5b74701849ed730f86695b4 [term 1 LEADER]: Becoming Leader. State: Replica: d9518af1d5b74701849ed730f86695b4, State: Running, Role: LEADER
07:29:31.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.855865 8548 consensus_queue.cc:237] T 00000000000000000000000000000000 P d9518af1d5b74701849ed730f86695b4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c25a52d16dc24d558d31e61573570268" member_type: VOTER last_known_addr { host: "127.8.28.62" port: 45427 } } peers { permanent_uuid: "ab6b10d629bc47bd902467fd80dc9d37" member_type: VOTER last_known_addr { host: "127.8.28.61" port: 34477 } } peers { permanent_uuid: "d9518af1d5b74701849ed730f86695b4" member_type: VOTER last_known_addr { host: "127.8.28.60" port: 39483 } }
07:29:31.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.867105 8568 sys_catalog.cc:455] T 00000000000000000000000000000000 P d9518af1d5b74701849ed730f86695b4 [sys.catalog]: SysCatalogTable state changed. Reason: New leader d9518af1d5b74701849ed730f86695b4. Latest consensus state: current_term: 1 leader_uuid: "d9518af1d5b74701849ed730f86695b4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c25a52d16dc24d558d31e61573570268" member_type: VOTER last_known_addr { host: "127.8.28.62" port: 45427 } } peers { permanent_uuid: "ab6b10d629bc47bd902467fd80dc9d37" member_type: VOTER last_known_addr { host: "127.8.28.61" port: 34477 } } peers { permanent_uuid: "d9518af1d5b74701849ed730f86695b4" member_type: VOTER last_known_addr { host: "127.8.28.60" port: 39483 } } }
07:29:31.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.867842 8568 sys_catalog.cc:458] T 00000000000000000000000000000000 P d9518af1d5b74701849ed730f86695b4 [sys.catalog]: This master's current role is: LEADER
07:29:31.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.874359 8571 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
07:29:31.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.889868 8571 catalog_manager.cc:1484] Initializing Kudu cluster ID...
07:29:31.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.913045 8434 raft_consensus.cc:1261] T 00000000000000000000000000000000 P ab6b10d629bc47bd902467fd80dc9d37 [term 1 FOLLOWER]: Refusing update from remote peer d9518af1d5b74701849ed730f86695b4: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
07:29:31.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.914598 8568 consensus_queue.cc:1035] T 00000000000000000000000000000000 P d9518af1d5b74701849ed730f86695b4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ab6b10d629bc47bd902467fd80dc9d37" member_type: VOTER last_known_addr { host: "127.8.28.61" port: 34477 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
07:29:31.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:31.936676 8578 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ab6b10d629bc47bd902467fd80dc9d37: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:29:31.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:31.937043 8578 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:29:31.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.907876 8386 master_runner.cc:186] Error getting master registration for 127.8.28.62:45427: OK, instance_id { permanent_uuid: "c25a52d16dc24d558d31e61573570268" instance_seqno: 1790234967556974 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
07:29:31.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:31.984038 8386 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:29:32.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:32.012117 8563 sys_catalog.cc:455] T 00000000000000000000000000000000 P ab6b10d629bc47bd902467fd80dc9d37 [sys.catalog]: SysCatalogTable state changed. Reason: New leader d9518af1d5b74701849ed730f86695b4. Latest consensus state: current_term: 1 leader_uuid: "d9518af1d5b74701849ed730f86695b4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c25a52d16dc24d558d31e61573570268" member_type: VOTER last_known_addr { host: "127.8.28.62" port: 45427 } } peers { permanent_uuid: "ab6b10d629bc47bd902467fd80dc9d37" member_type: VOTER last_known_addr { host: "127.8.28.61" port: 34477 } } peers { permanent_uuid: "d9518af1d5b74701849ed730f86695b4" member_type: VOTER last_known_addr { host: "127.8.28.60" port: 39483 } } }
07:29:32.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:32.012876 8563 sys_catalog.cc:458] T 00000000000000000000000000000000 P ab6b10d629bc47bd902467fd80dc9d37 [sys.catalog]: This master's current role is: FOLLOWER
07:29:32.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:32.013753 8577 mvcc.cc:205] Tried to move back new op lower bound from 7332802444925980672 to 7332802444732489728. Current Snapshot: MvccSnapshot[applied={T|T < 7332802444925980672}]
07:29:32.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:32.019418 8580 mvcc.cc:205] Tried to move back new op lower bound from 7332802444925980672 to 7332802444732489728. Current Snapshot: MvccSnapshot[applied={T|T < 7332802444925980672}]
07:29:32.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:32.026049 8548 sys_catalog.cc:455] T 00000000000000000000000000000000 P d9518af1d5b74701849ed730f86695b4 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "d9518af1d5b74701849ed730f86695b4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c25a52d16dc24d558d31e61573570268" member_type: VOTER last_known_addr { host: "127.8.28.62" port: 45427 } } peers { permanent_uuid: "ab6b10d629bc47bd902467fd80dc9d37" member_type: VOTER last_known_addr { host: "127.8.28.61" port: 34477 } } peers { permanent_uuid: "d9518af1d5b74701849ed730f86695b4" member_type: VOTER last_known_addr { host: "127.8.28.60" port: 39483 } } }
07:29:32.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:32.026790 8548 sys_catalog.cc:458] T 00000000000000000000000000000000 P d9518af1d5b74701849ed730f86695b4 [sys.catalog]: This master's current role is: LEADER
07:29:32.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:32.027904 8563 sys_catalog.cc:455] T 00000000000000000000000000000000 P ab6b10d629bc47bd902467fd80dc9d37 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "d9518af1d5b74701849ed730f86695b4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c25a52d16dc24d558d31e61573570268" member_type: VOTER last_known_addr { host: "127.8.28.62" port: 45427 } } peers { permanent_uuid: "ab6b10d629bc47bd902467fd80dc9d37" member_type: VOTER last_known_addr { host: "127.8.28.61" port: 34477 } } peers { permanent_uuid: "d9518af1d5b74701849ed730f86695b4" member_type: VOTER last_known_addr { host: "127.8.28.60" port: 39483 } } }
07:29:32.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:32.028882 8563 sys_catalog.cc:458] T 00000000000000000000000000000000 P ab6b10d629bc47bd902467fd80dc9d37 [sys.catalog]: This master's current role is: FOLLOWER
07:29:32.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:32.030689 8571 catalog_manager.cc:1347] Generated new cluster ID: c0f5e4cdec22433b962a86b75fbfb632
07:29:32.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:32.030961 8571 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
07:29:32.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:32.060824 8571 catalog_manager.cc:1370] Generated new certificate authority record
07:29:32.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:32.066233 8571 catalog_manager.cc:1504] Loading token signing keys...
07:29:32.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:32.097261 8571 catalog_manager.cc:5868] T 00000000000000000000000000000000 P d9518af1d5b74701849ed730f86695b4: Generated new TSK 0
07:29:32.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:32.098640 8571 catalog_manager.cc:1514] Initializing in-progress tserver states...
07:29:32.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:32.141495 8304 external_mini_cluster.cc:1326] Running /tmp/dist-test-task8eqvEo/build/tsan/bin/kudu
07:29:32.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task8eqvEo/build/tsan/bin/kudu
07:29:32.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/ts-0/wal
07:29:32.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/ts-0/data
07:29:32.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:29:32.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:29:32.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:29:32.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:29:32.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:29:32.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:29:32.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/ts-0/logs
07:29:32.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/ts-0/data/info.pb
07:29:32.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:29:32.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:29:32.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:29:32.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:29:32.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:29:32.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:29:32.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:29:32.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:29:32.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
07:29:32.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:29:32.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.28.1:0
07:29:32.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.28.1
07:29:32.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.28.1
07:29:32.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:29:32.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.28.62:45427,127.8.28.61:34477,127.8.28.60:39483
07:29:32.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.28.20:40953
07:29:32.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:29:32.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:29:32.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
07:29:32.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:32.226019 8384 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.28.60" port: 39483 } attempt: 7
07:29:32.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:32.308339 8384 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c25a52d16dc24d558d31e61573570268: Bootstrap starting.
07:29:32.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:32.316442 8384 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P c25a52d16dc24d558d31e61573570268: Neither blocks nor log segments found. Creating new log.
07:29:32.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:32.319352 8384 log.cc:824] T 00000000000000000000000000000000 P c25a52d16dc24d558d31e61573570268: Log is configured to *not* fsync() on all Append() calls
07:29:32.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:32.331261 8384 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c25a52d16dc24d558d31e61573570268: No bootstrap required, opened a new log
07:29:32.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:32.369556 8384 raft_consensus.cc:348] T 00000000000000000000000000000000 P c25a52d16dc24d558d31e61573570268 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c25a52d16dc24d558d31e61573570268" member_type: VOTER last_known_addr { host: "127.8.28.62" port: 45427 } } peers { permanent_uuid: "ab6b10d629bc47bd902467fd80dc9d37" member_type: VOTER last_known_addr { host: "127.8.28.61" port: 34477 } } peers { permanent_uuid: "d9518af1d5b74701849ed730f86695b4" member_type: VOTER last_known_addr { host: "127.8.28.60" port: 39483 } }
07:29:32.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:32.370873 8384 raft_consensus.cc:374] T 00000000000000000000000000000000 P c25a52d16dc24d558d31e61573570268 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:29:32.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:32.371486 8384 raft_consensus.cc:729] T 00000000000000000000000000000000 P c25a52d16dc24d558d31e61573570268 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c25a52d16dc24d558d31e61573570268, State: Initialized, Role: FOLLOWER
07:29:32.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:32.372684 8384 consensus_queue.cc:260] T 00000000000000000000000000000000 P c25a52d16dc24d558d31e61573570268 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c25a52d16dc24d558d31e61573570268" member_type: VOTER last_known_addr { host: "127.8.28.62" port: 45427 } } peers { permanent_uuid: "ab6b10d629bc47bd902467fd80dc9d37" member_type: VOTER last_known_addr { host: "127.8.28.61" port: 34477 } } peers { permanent_uuid: "d9518af1d5b74701849ed730f86695b4" member_type: VOTER last_known_addr { host: "127.8.28.60" port: 39483 } }
07:29:32.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:32.385401 8588 sys_catalog.cc:455] T 00000000000000000000000000000000 P c25a52d16dc24d558d31e61573570268 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c25a52d16dc24d558d31e61573570268" member_type: VOTER last_known_addr { host: "127.8.28.62" port: 45427 } } peers { permanent_uuid: "ab6b10d629bc47bd902467fd80dc9d37" member_type: VOTER last_known_addr { host: "127.8.28.61" port: 34477 } } peers { permanent_uuid: "d9518af1d5b74701849ed730f86695b4" member_type: VOTER last_known_addr { host: "127.8.28.60" port: 39483 } } }
07:29:32.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:32.390398 8588 sys_catalog.cc:458] T 00000000000000000000000000000000 P c25a52d16dc24d558d31e61573570268 [sys.catalog]: This master's current role is: FOLLOWER
07:29:32.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:32.391366 8357 raft_consensus.cc:3037] T 00000000000000000000000000000000 P c25a52d16dc24d558d31e61573570268 [term 0 FOLLOWER]: Advancing to term 1
07:29:32.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:32.394965 8359 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d9518af1d5b74701849ed730f86695b4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c25a52d16dc24d558d31e61573570268" is_pre_election: true
07:29:32.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:32.390261 8358 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d9518af1d5b74701849ed730f86695b4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c25a52d16dc24d558d31e61573570268"
07:29:32.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:32.390046 8384 sys_catalog.cc:564] T 00000000000000000000000000000000 P c25a52d16dc24d558d31e61573570268 [sys.catalog]: configured and running, proceeding with master startup.
07:29:32.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:32.409507 8357 raft_consensus.cc:1261] T 00000000000000000000000000000000 P c25a52d16dc24d558d31e61573570268 [term 1 FOLLOWER]: Refusing update from remote peer d9518af1d5b74701849ed730f86695b4: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
07:29:32.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:32.412122 8584 consensus_queue.cc:1035] T 00000000000000000000000000000000 P d9518af1d5b74701849ed730f86695b4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c25a52d16dc24d558d31e61573570268" member_type: VOTER last_known_addr { host: "127.8.28.62" port: 45427 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
07:29:32.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:32.488624 8588 sys_catalog.cc:455] T 00000000000000000000000000000000 P c25a52d16dc24d558d31e61573570268 [sys.catalog]: SysCatalogTable state changed. Reason: New leader d9518af1d5b74701849ed730f86695b4. Latest consensus state: current_term: 1 leader_uuid: "d9518af1d5b74701849ed730f86695b4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c25a52d16dc24d558d31e61573570268" member_type: VOTER last_known_addr { host: "127.8.28.62" port: 45427 } } peers { permanent_uuid: "ab6b10d629bc47bd902467fd80dc9d37" member_type: VOTER last_known_addr { host: "127.8.28.61" port: 34477 } } peers { permanent_uuid: "d9518af1d5b74701849ed730f86695b4" member_type: VOTER last_known_addr { host: "127.8.28.60" port: 39483 } } }
07:29:32.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:32.490778 8594 sys_catalog.cc:455] T 00000000000000000000000000000000 P c25a52d16dc24d558d31e61573570268 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "d9518af1d5b74701849ed730f86695b4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c25a52d16dc24d558d31e61573570268" member_type: VOTER last_known_addr { host: "127.8.28.62" port: 45427 } } peers { permanent_uuid: "ab6b10d629bc47bd902467fd80dc9d37" member_type: VOTER last_known_addr { host: "127.8.28.61" port: 34477 } } peers { permanent_uuid: "d9518af1d5b74701849ed730f86695b4" member_type: VOTER last_known_addr { host: "127.8.28.60" port: 39483 } } }
07:29:32.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:32.491648 8594 sys_catalog.cc:458] T 00000000000000000000000000000000 P c25a52d16dc24d558d31e61573570268 [sys.catalog]: This master's current role is: FOLLOWER
07:29:32.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:32.492580 8588 sys_catalog.cc:458] T 00000000000000000000000000000000 P c25a52d16dc24d558d31e61573570268 [sys.catalog]: This master's current role is: FOLLOWER
07:29:32.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:32.536862 8604 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
07:29:32.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:32.775612 8585 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:29:32.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:32.776232 8585 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:29:32.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:32.777012 8585 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:29:32.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:32.831568 8585 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:29:32.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:32.832821 8585 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.8.28.1
07:29:32.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:32.893885 8585 tablet_server_runner.cc:78] Tablet server non-default flags:
07:29:32.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:29:32.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.28.20:40953
07:29:32.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:29:32.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:29:32.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/ts-0/data
07:29:32.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/ts-0/wal
07:29:32.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:29:32.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:29:32.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.28.1:0
07:29:32.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:29:32.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:29:32.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:29:32.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/ts-0/data/info.pb
07:29:32.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.28.1
07:29:32.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:29:32.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.28.62:45427,127.8.28.61:34477,127.8.28.60:39483
07:29:32.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:29:32.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:29:32.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:29:32.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:29:32.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:29:32.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.28.1
07:29:32.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/ts-0/logs
07:29:32.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:29:32.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:29:32.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:29:32.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
07:29:32.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:29:32.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:29:32.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:29:32.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 24 Sep 2026 06:42:32 UTC on ccycloud.quasar-lgarko.root.comops.site
07:29:32.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4669406
07:29:32.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:29:32.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:32.895430 8585 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:29:32.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:32.897393 8585 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:29:32.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:32.923224 8610 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:29:32.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:32.942567 8578 catalog_manager.cc:1259] Loaded cluster ID: c0f5e4cdec22433b962a86b75fbfb632
07:29:32.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:32.943172 8578 catalog_manager.cc:1552] T 00000000000000000000000000000000 P ab6b10d629bc47bd902467fd80dc9d37: loading cluster ID for follower catalog manager: success
07:29:32.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:32.951457 8578 catalog_manager.cc:1574] T 00000000000000000000000000000000 P ab6b10d629bc47bd902467fd80dc9d37: acquiring CA information for follower catalog manager: success
07:29:32.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:32.963151 8584 sys_catalog.cc:455] T 00000000000000000000000000000000 P d9518af1d5b74701849ed730f86695b4 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "d9518af1d5b74701849ed730f86695b4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c25a52d16dc24d558d31e61573570268" member_type: VOTER last_known_addr { host: "127.8.28.62" port: 45427 } } peers { permanent_uuid: "ab6b10d629bc47bd902467fd80dc9d37" member_type: VOTER last_known_addr { host: "127.8.28.61" port: 34477 } } peers { permanent_uuid: "d9518af1d5b74701849ed730f86695b4" member_type: VOTER last_known_addr { host: "127.8.28.60" port: 39483 } } }
07:29:32.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:32.963881 8584 sys_catalog.cc:458] T 00000000000000000000000000000000 P d9518af1d5b74701849ed730f86695b4 [sys.catalog]: This master's current role is: LEADER
07:29:32.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:32.968823 8578 catalog_manager.cc:1602] T 00000000000000000000000000000000 P ab6b10d629bc47bd902467fd80dc9d37: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
07:29:33.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:33.567791 8604 catalog_manager.cc:1259] Loaded cluster ID: c0f5e4cdec22433b962a86b75fbfb632
07:29:33.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:33.569347 8604 catalog_manager.cc:1552] T 00000000000000000000000000000000 P c25a52d16dc24d558d31e61573570268: loading cluster ID for follower catalog manager: success
07:29:33.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:33.606987 8604 catalog_manager.cc:1574] T 00000000000000000000000000000000 P c25a52d16dc24d558d31e61573570268: acquiring CA information for follower catalog manager: success
07:29:33.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:33.634045 8604 catalog_manager.cc:1602] T 00000000000000000000000000000000 P c25a52d16dc24d558d31e61573570268: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
07:29:35.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:34.316994 8609 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 8585
07:29:35.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:35.069325 8609 kernel_stack_watchdog.cc:198] Thread 8585 stuck at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:637 for 380ms:
07:29:35.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
07:29:35.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
07:29:35.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
07:29:35.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
07:29:35.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:35.071592 8585 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 2.138s user 0.000s sys 0.002s
07:29:35.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:35.071952 8585 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 2.149s user 0.000s sys 0.002s
07:29:35.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:32.923229 8611 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:29:35.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:35.074618 8614 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:29:35.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:35.077322 8612 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2149 milliseconds
07:29:35.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:35.077406 8585 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
07:29:35.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:35.078898 8585 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:29:35.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:35.082039 8585 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:29:35.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:35.083469 8585 hybrid_clock.cc:648] HybridClock initialized: now 1790234975083415 us; error 68 us; skew 500 ppm
07:29:35.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:35.084512 8585 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:29:35.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:35.098353 8585 webserver.cc:460] Webserver started at http://127.8.28.1:43601/ using document root <none> and password file <none>
07:29:35.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:35.099577 8585 fs_manager.cc:359] Metadata directory not provided
07:29:35.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:35.099848 8585 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:29:35.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:35.100396 8585 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:29:35.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:35.107443 8585 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/ts-0/data/instance:
07:29:35.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "795d8344cced48e386b2d7de6ebb2bb6"
07:29:35.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:29:35 on dist-test-slave-hp5j"
07:29:35.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:35.108956 8585 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/ts-0/wal/instance:
07:29:35.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "795d8344cced48e386b2d7de6ebb2bb6"
07:29:35.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:29:35 on dist-test-slave-hp5j"
07:29:35.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:35.118639 8585 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.007s sys 0.001s
07:29:35.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:35.127199 8625 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:29:35.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:35.128562 8585 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.004s sys 0.001s
07:29:35.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:35.128945 8585 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/ts-0/data,/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/ts-0/wal
07:29:35.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "795d8344cced48e386b2d7de6ebb2bb6"
07:29:35.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:29:35 on dist-test-slave-hp5j"
07:29:35.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:35.129357 8585 fs_report.cc:383] FS layout report
07:29:35.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:29:35.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/ts-0/wal
07:29:35.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/ts-0/wal
07:29:35.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/ts-0/data/data
07:29:35.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:29:35.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:29:35.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:29:35.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:29:35.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:29:35.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:29:35.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:29:35.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:29:35.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:29:35.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:29:35.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:29:35.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:35.177966 8585 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:29:35.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:35.179348 8585 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:29:35.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:35.179921 8585 kserver.cc:163] Server-wide thread pool size limit: 3276
07:29:35.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:35.182564 8585 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
07:29:35.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:35.187978 8585 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
07:29:35.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:35.188265 8585 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
07:29:35.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:35.188465 8585 ts_tablet_manager.cc:610] Registered 0 tablets
07:29:35.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:35.188591 8585 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
07:29:35.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:35.375756 8585 rpc_server.cc:307] RPC server started. Bound to: 127.8.28.1:35079
07:29:35.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:35.375891 8737 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.8.28.1:35079 every 8 connection(s)
07:29:35.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:35.378685 8585 server_base.cc:1166] Dumped server information to /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/ts-0/data/info.pb
07:29:35.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:35.381963 8304 external_mini_cluster.cc:1388] Started /tmp/dist-test-task8eqvEo/build/tsan/bin/kudu as pid 8585
07:29:35.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:35.382419 8304 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/ts-0/wal/instance
07:29:35.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:35.409886 8304 external_mini_cluster.cc:1326] Running /tmp/dist-test-task8eqvEo/build/tsan/bin/kudu
07:29:35.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task8eqvEo/build/tsan/bin/kudu
07:29:35.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/ts-1/wal
07:29:35.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/ts-1/data
07:29:35.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:29:35.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:29:35.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:29:35.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:29:35.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:29:35.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:29:35.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/ts-1/logs
07:29:35.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/ts-1/data/info.pb
07:29:35.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:29:35.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:29:35.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:29:35.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:29:35.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:29:35.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:29:35.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:29:35.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:29:35.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
07:29:35.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:29:35.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.28.2:0
07:29:35.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.28.2
07:29:35.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.28.2
07:29:35.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:29:35.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.28.62:45427,127.8.28.61:34477,127.8.28.60:39483
07:29:35.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.28.20:40953
07:29:35.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:29:35.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:29:35.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
07:29:35.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:35.464805 8738 heartbeater.cc:347] Connected to a master server at 127.8.28.60:39483
07:29:35.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:35.465360 8738 heartbeater.cc:464] Registering TS with master...
07:29:35.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:35.466886 8738 heartbeater.cc:511] Master 127.8.28.60:39483 requested a full tablet report, sending...
07:29:35.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:35.470289 8502 ts_manager.cc:194] Registered new tserver with Master: 795d8344cced48e386b2d7de6ebb2bb6 (127.8.28.1:35079)
07:29:35.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:35.470978 8739 heartbeater.cc:347] Connected to a master server at 127.8.28.61:34477
07:29:35.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:35.471743 8739 heartbeater.cc:464] Registering TS with master...
07:29:35.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:35.472538 8739 heartbeater.cc:511] Master 127.8.28.61:34477 requested a full tablet report, sending...
07:29:35.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:35.473534 8502 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.8.28.1:52225
07:29:35.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:35.476750 8424 ts_manager.cc:194] Registered new tserver with Master: 795d8344cced48e386b2d7de6ebb2bb6 (127.8.28.1:35079)
07:29:35.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:35.494500 8743 heartbeater.cc:347] Connected to a master server at 127.8.28.62:45427
07:29:35.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:35.494879 8743 heartbeater.cc:464] Registering TS with master...
07:29:35.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:35.495602 8743 heartbeater.cc:511] Master 127.8.28.62:45427 requested a full tablet report, sending...
07:29:35.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:35.498744 8349 ts_manager.cc:194] Registered new tserver with Master: 795d8344cced48e386b2d7de6ebb2bb6 (127.8.28.1:35079)
07:29:35.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:35.814193 8742 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:29:35.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:35.814882 8742 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:29:35.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:35.815411 8742 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:29:35.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:35.849641 8742 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:29:35.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:35.850600 8742 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.8.28.2
07:29:35.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:35.887871 8742 tablet_server_runner.cc:78] Tablet server non-default flags:
07:29:35.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:29:35.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.28.20:40953
07:29:35.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:29:35.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:29:35.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/ts-1/data
07:29:35.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/ts-1/wal
07:29:35.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:29:35.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:29:35.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.28.2:0
07:29:35.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:29:35.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:29:35.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:29:35.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/ts-1/data/info.pb
07:29:35.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.28.2
07:29:35.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:29:35.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.28.62:45427,127.8.28.61:34477,127.8.28.60:39483
07:29:35.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:29:35.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:29:35.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:29:35.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:29:35.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:29:35.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.28.2
07:29:35.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/ts-1/logs
07:29:35.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:29:35.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:29:35.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:29:35.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
07:29:35.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:29:35.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:29:35.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:29:35.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 24 Sep 2026 06:42:32 UTC on ccycloud.quasar-lgarko.root.comops.site
07:29:35.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4669406
07:29:35.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:29:35.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:35.889150 8742 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:29:35.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:35.890887 8742 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:29:35.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:35.916965 8757 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:29:36.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:36.480141 8738 heartbeater.cc:503] Master 127.8.28.60:39483 was elected leader, sending a full tablet report...
07:29:37.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:35.916954 8756 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:29:37.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:35.918802 8742 server_base.cc:1034] running on GCE node
07:29:37.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:35.918938 8759 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:29:37.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:37.177060 8742 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:29:37.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:37.179486 8742 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:29:37.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:37.180872 8742 hybrid_clock.cc:648] HybridClock initialized: now 1790234977180842 us; error 44 us; skew 500 ppm
07:29:37.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:37.181743 8742 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:29:37.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:37.189967 8742 webserver.cc:460] Webserver started at http://127.8.28.2:34731/ using document root <none> and password file <none>
07:29:37.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:37.191466 8742 fs_manager.cc:359] Metadata directory not provided
07:29:37.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:37.191789 8742 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:29:37.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:37.192618 8742 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:29:37.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:37.203091 8742 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/ts-1/data/instance:
07:29:37.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0107b52a518448cf9ab6f7e76faf5e6c"
07:29:37.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:29:37 on dist-test-slave-hp5j"
07:29:37.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:37.205680 8742 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/ts-1/wal/instance:
07:29:37.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0107b52a518448cf9ab6f7e76faf5e6c"
07:29:37.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:29:37 on dist-test-slave-hp5j"
07:29:37.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:37.221467 8742 fs_manager.cc:691] Time spent creating directory manager: real 0.014s user 0.013s sys 0.000s
07:29:37.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:37.232038 8767 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:29:37.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:37.233538 8742 fs_manager.cc:725] Time spent opening block manager: real 0.006s user 0.006s sys 0.001s
07:29:37.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:37.234207 8742 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/ts-1/data,/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/ts-1/wal
07:29:37.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0107b52a518448cf9ab6f7e76faf5e6c"
07:29:37.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:29:37 on dist-test-slave-hp5j"
07:29:37.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:37.235189 8742 fs_report.cc:383] FS layout report
07:29:37.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:29:37.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/ts-1/wal
07:29:37.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/ts-1/wal
07:29:37.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/ts-1/data/data
07:29:37.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:29:37.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:29:37.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:29:37.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:29:37.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:29:37.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:29:37.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:29:37.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:29:37.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:29:37.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:29:37.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:29:37.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:37.299243 8742 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:29:37.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:37.300786 8742 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:29:37.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:37.301497 8742 kserver.cc:163] Server-wide thread pool size limit: 3276
07:29:37.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:37.304481 8742 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
07:29:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:37.308933 8742 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
07:29:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:37.309160 8742 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
07:29:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:37.309511 8742 ts_tablet_manager.cc:610] Registered 0 tablets
07:29:37.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:37.309733 8742 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
07:29:37.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:37.472520 8742 rpc_server.cc:307] RPC server started. Bound to: 127.8.28.2:43715
07:29:37.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:37.472618 8879 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.8.28.2:43715 every 8 connection(s)
07:29:37.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:37.475049 8742 server_base.cc:1166] Dumped server information to /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/ts-1/data/info.pb
07:29:37.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:37.476015 8304 external_mini_cluster.cc:1388] Started /tmp/dist-test-task8eqvEo/build/tsan/bin/kudu as pid 8742
07:29:37.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:37.477037 8304 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/ts-1/wal/instance
07:29:37.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:37.505228 8304 external_mini_cluster.cc:1326] Running /tmp/dist-test-task8eqvEo/build/tsan/bin/kudu
07:29:37.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task8eqvEo/build/tsan/bin/kudu
07:29:37.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/ts-2/wal
07:29:37.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/ts-2/data
07:29:37.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:29:37.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:29:37.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:29:37.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:29:37.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:29:37.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:29:37.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/ts-2/logs
07:29:37.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/ts-2/data/info.pb
07:29:37.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:29:37.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:29:37.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:29:37.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:29:37.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:29:37.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:29:37.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:29:37.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:29:37.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
07:29:37.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:29:37.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.28.3:0
07:29:37.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.28.3
07:29:37.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.28.3
07:29:37.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:29:37.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.28.62:45427,127.8.28.61:34477,127.8.28.60:39483
07:29:37.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.28.20:40953
07:29:37.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:29:37.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:29:37.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
07:29:37.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:37.537832 8881 heartbeater.cc:347] Connected to a master server at 127.8.28.61:34477
07:29:37.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:37.538455 8881 heartbeater.cc:464] Registering TS with master...
07:29:37.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:37.540000 8881 heartbeater.cc:511] Master 127.8.28.61:34477 requested a full tablet report, sending...
07:29:37.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:37.543156 8424 ts_manager.cc:194] Registered new tserver with Master: 0107b52a518448cf9ab6f7e76faf5e6c (127.8.28.2:43715)
07:29:37.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:37.550431 8880 heartbeater.cc:347] Connected to a master server at 127.8.28.60:39483
07:29:37.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:37.550813 8880 heartbeater.cc:464] Registering TS with master...
07:29:37.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:37.551606 8880 heartbeater.cc:511] Master 127.8.28.60:39483 requested a full tablet report, sending...
07:29:37.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:37.554268 8882 heartbeater.cc:347] Connected to a master server at 127.8.28.62:45427
07:29:37.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:37.554356 8502 ts_manager.cc:194] Registered new tserver with Master: 0107b52a518448cf9ab6f7e76faf5e6c (127.8.28.2:43715)
07:29:37.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:37.554822 8882 heartbeater.cc:464] Registering TS with master...
07:29:37.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:37.555591 8882 heartbeater.cc:511] Master 127.8.28.62:45427 requested a full tablet report, sending...
07:29:37.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:37.556332 8502 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.8.28.2:51049
07:29:37.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:37.558399 8349 ts_manager.cc:194] Registered new tserver with Master: 0107b52a518448cf9ab6f7e76faf5e6c (127.8.28.2:43715)
07:29:37.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:37.857106 8887 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:29:37.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:37.857686 8887 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:29:37.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:37.858280 8887 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:29:37.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:37.890303 8887 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:29:37.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:37.891260 8887 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.8.28.3
07:29:37.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:37.937562 8887 tablet_server_runner.cc:78] Tablet server non-default flags:
07:29:37.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:29:37.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.28.20:40953
07:29:37.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:29:37.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:29:37.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/ts-2/data
07:29:37.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/ts-2/wal
07:29:37.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:29:37.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:29:37.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.28.3:0
07:29:37.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:29:37.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:29:37.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:29:37.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/ts-2/data/info.pb
07:29:37.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.28.3
07:29:37.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:29:37.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.28.62:45427,127.8.28.61:34477,127.8.28.60:39483
07:29:37.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:29:37.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:29:37.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:29:37.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:29:37.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:29:37.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.28.3
07:29:37.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/ts-2/logs
07:29:37.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:29:37.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:29:37.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:29:37.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
07:29:37.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:29:37.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:29:37.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:29:37.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 24 Sep 2026 06:42:32 UTC on ccycloud.quasar-lgarko.root.comops.site
07:29:37.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4669406
07:29:37.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:29:37.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:37.939109 8887 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:29:37.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:37.940757 8887 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:29:37.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:37.962236 8899 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:29:38.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:38.561342 8880 heartbeater.cc:503] Master 127.8.28.60:39483 was elected leader, sending a full tablet report...
07:29:39.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:37.963873 8887 server_base.cc:1034] running on GCE node
07:29:39.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:37.963157 8901 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:29:39.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:37.962229 8898 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:29:39.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:39.175518 8887 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:29:39.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:39.178596 8887 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:29:39.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:39.180379 8887 hybrid_clock.cc:648] HybridClock initialized: now 1790234979180323 us; error 72 us; skew 500 ppm
07:29:39.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:39.181175 8887 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:29:39.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:39.188206 8887 webserver.cc:460] Webserver started at http://127.8.28.3:33993/ using document root <none> and password file <none>
07:29:39.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:39.189077 8887 fs_manager.cc:359] Metadata directory not provided
07:29:39.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:39.189276 8887 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:29:39.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:39.189721 8887 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:29:39.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:39.194455 8887 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/ts-2/data/instance:
07:29:39.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9e384ecc086449978a6e1ef474c11d90"
07:29:39.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:29:39 on dist-test-slave-hp5j"
07:29:39.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:39.195588 8887 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/ts-2/wal/instance:
07:29:39.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9e384ecc086449978a6e1ef474c11d90"
07:29:39.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:29:39 on dist-test-slave-hp5j"
07:29:39.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:39.204131 8887 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.005s sys 0.004s
07:29:39.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:39.211511 8909 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:29:39.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:39.212862 8887 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.005s sys 0.000s
07:29:39.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:39.213267 8887 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/ts-2/data,/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/ts-2/wal
07:29:39.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9e384ecc086449978a6e1ef474c11d90"
07:29:39.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:29:39 on dist-test-slave-hp5j"
07:29:39.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:39.213686 8887 fs_report.cc:383] FS layout report
07:29:39.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:29:39.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/ts-2/wal
07:29:39.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/ts-2/wal
07:29:39.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/ts-2/data/data
07:29:39.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:29:39.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:29:39.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:29:39.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:29:39.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:29:39.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:29:39.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:29:39.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:29:39.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:29:39.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:29:39.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:29:39.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:39.260728 8887 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:29:39.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:39.262635 8887 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:29:39.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:39.263207 8887 kserver.cc:163] Server-wide thread pool size limit: 3276
07:29:39.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:39.265893 8887 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
07:29:39.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:39.270200 8887 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
07:29:39.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:39.270474 8887 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
07:29:39.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:39.270774 8887 ts_tablet_manager.cc:610] Registered 0 tablets
07:29:39.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:39.270967 8887 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
07:29:39.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:39.435467 8887 rpc_server.cc:307] RPC server started. Bound to: 127.8.28.3:44725
07:29:39.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:39.435557 9021 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.8.28.3:44725 every 8 connection(s)
07:29:39.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:39.437832 8887 server_base.cc:1166] Dumped server information to /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/ts-2/data/info.pb
07:29:39.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:39.443027 8304 external_mini_cluster.cc:1388] Started /tmp/dist-test-task8eqvEo/build/tsan/bin/kudu as pid 8887
07:29:39.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:39.443629 8304 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster1168659196521048114/ts-2/wal/instance
07:29:39.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:39.526697 9022 heartbeater.cc:347] Connected to a master server at 127.8.28.60:39483
07:29:39.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:39.527271 9022 heartbeater.cc:464] Registering TS with master...
07:29:39.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:39.528726 9022 heartbeater.cc:511] Master 127.8.28.60:39483 requested a full tablet report, sending...
07:29:39.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:39.531925 8502 ts_manager.cc:194] Registered new tserver with Master: 9e384ecc086449978a6e1ef474c11d90 (127.8.28.3:44725)
07:29:39.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:39.531078 9023 heartbeater.cc:347] Connected to a master server at 127.8.28.61:34477
07:29:39.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:39.532408 9023 heartbeater.cc:464] Registering TS with master...
07:29:39.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:39.533411 9023 heartbeater.cc:511] Master 127.8.28.61:34477 requested a full tablet report, sending...
07:29:39.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:39.535247 8502 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.8.28.3:51409
07:29:39.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:39.536661 8424 ts_manager.cc:194] Registered new tserver with Master: 9e384ecc086449978a6e1ef474c11d90 (127.8.28.3:44725)
07:29:39.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:39.537194 9024 heartbeater.cc:347] Connected to a master server at 127.8.28.62:45427
07:29:39.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:39.538292 9024 heartbeater.cc:464] Registering TS with master...
07:29:39.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:39.539286 9024 heartbeater.cc:511] Master 127.8.28.62:45427 requested a full tablet report, sending...
07:29:39.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:39.542938 8349 ts_manager.cc:194] Registered new tserver with Master: 9e384ecc086449978a6e1ef474c11d90 (127.8.28.3:44725)
07:29:39.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:39.553932 8304 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
07:29:39.565 [DEBUG - main] (MiniKuduCluster.java:195) Response:
07:29:39.567 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
07:29:39.579 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.8.28.62"
port: 45427
}
bound_http_address {
host: "127.8.28.62"
port: 38819
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.8.28.61"
port: 34477
}
bound_http_address {
host: "127.8.28.61"
port: 44553
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.8.28.60"
port: 39483
}
bound_http_address {
host: "127.8.28.60"
port: 44655
}
}
}
07:29:39.585 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
07:29:39.590 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.8.28.1"
port: 35079
}
bound_http_address {
host: "127.8.28.1"
port: 43601
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.8.28.2"
port: 43715
}
bound_http_address {
host: "127.8.28.2"
port: 34731
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.8.28.3"
port: 44725
}
bound_http_address {
host: "127.8.28.3"
port: 33993
}
}
}
07:29:39.590 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
07:29:39.802 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
07:29:39.910 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task8eqvEo/test-tmp
07:29:39.911 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
07:29:39.911 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task8eqvEo/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
07:29:39.914 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
07:29:40.252 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:29:40.252 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
07:29:40.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:40.255688 9034 mini_oidc.cc:152] Starting JWKS server
07:29:40.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:40.259364 9034 webserver.cc:460] Webserver started at http://127.0.0.1:44433/ using document root <none> and password file <none>
07:29:40.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:40.259825 9034 mini_oidc.cc:189] Starting OIDC Discovery server
07:29:40.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:40.262002 9034 webserver.cc:460] Webserver started at http://127.0.0.1:44935/ using document root <none> and password file <none>
07:29:40.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
07:29:40.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/krb5kdc/principal' for realm 'KRBTEST.COM',
07:29:40.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
07:29:40.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:29:40 dist-test-slave-hp5j krb5kdc[9046](info): setting up network...
07:29:40.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
07:29:40.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:29:40 dist-test-slave-hp5j krb5kdc[9046](info): set up 2 sockets
07:29:40.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:29:40 dist-test-slave-hp5j krb5kdc[9046](info): commencing operation
07:29:40.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
07:29:40.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:40.541636 9022 heartbeater.cc:503] Master 127.8.28.60:39483 was elected leader, sending a full tablet report...
07:29:41.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:41.389910 9034 mini_kdc.cc:121] Time spent starting KDC: real 1.102s user 0.006s sys 0.017s
07:29:41.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
07:29:41.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
07:29:41.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
07:29:41.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
07:29:41.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
07:29:41.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
07:29:41.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
07:29:41.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
07:29:41.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
07:29:41.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
07:29:41.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/krb5kdc/test-user.keytab.
07:29:41.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/krb5kdc/test-user.keytab.
07:29:41.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:29:41 dist-test-slave-hp5j krb5kdc[9046](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790234981, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:29:41.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
07:29:41.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-24T07:29:41Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
07:29:41.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-24T07:29:41Z Disabled control of system clock
07:29:41.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.8.210.190@KRBTEST.COM; defaulting to no policy
07:29:41.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
07:29:41.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.8.210.190@KRBTEST.COM" created.
07:29:41.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
07:29:41.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.8.210.190 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/krb5kdc/kudu_127.8.210.190.keytab.
07:29:41.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.8.210.190 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/krb5kdc/kudu_127.8.210.190.keytab.
07:29:41.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:41.610381 9034 external_mini_cluster.cc:1326] Running /tmp/dist-test-task8eqvEo/build/tsan/bin/kudu
07:29:41.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task8eqvEo/build/tsan/bin/kudu
07:29:41.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-0/wal
07:29:41.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-0/data
07:29:41.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:29:41.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:29:41.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:29:41.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:29:41.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:29:41.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:29:41.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-0/logs
07:29:41.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-0/data/info.pb
07:29:41.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:29:41.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:29:41.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:29:41.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:29:41.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:29:41.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:29:41.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:29:41.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:29:41.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:29:41.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:29:41.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:29:41.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:29:41.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.210.190:36273
07:29:41.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.210.190
07:29:41.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:29:41.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/krb5kdc/kudu_127.8.210.190.keytab
07:29:41.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.210.190
07:29:41.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:29:41.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
07:29:41.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
07:29:41.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.210.148:43435
07:29:41.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:29:41.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:29:41.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:29:41.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:29:41.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.210.190:36273,127.8.210.189:45987,127.8.210.188:35193
07:29:41.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:29:41.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44433/jwks.json
07:29:41.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
07:29:41.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:41.927013 9059 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:29:41.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:41.927558 9059 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:29:41.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:41.928148 9059 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:29:41.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:41.959613 9059 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
07:29:41.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:41.959837 9059 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:29:41.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:41.960095 9059 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:29:41.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:41.960338 9059 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:29:41.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:41.960625 9059 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:29:41.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:41.960942 9059 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:29:41.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:41.961189 9059 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44433/jwks.json
07:29:42.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:42.003916 9059 master_runner.cc:386] Master server non-default flags:
07:29:42.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:29:42.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.210.148:43435
07:29:42.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:29:42.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:29:42.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-0/data
07:29:42.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-0/wal
07:29:42.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
07:29:42.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:29:42.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.210.190:36273,127.8.210.189:45987,127.8.210.188:35193
07:29:42.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/krb5kdc/kudu_127.8.210.190.keytab
07:29:42.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.210.190
07:29:42.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:29:42.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:29:42.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:29:42.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.210.190:36273
07:29:42.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:29:42.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:29:42.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:29:42.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44433/jwks.json
07:29:42.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:29:42.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
07:29:42.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
07:29:42.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:29:42.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:29:42.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-0/data/info.pb
07:29:42.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.210.190
07:29:42.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:29:42.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:29:42.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:29:42.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:29:42.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:29:42.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:29:42.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-0/logs
07:29:42.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:29:42.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:29:42.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:29:42.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:29:42.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:29:42.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:29:42.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:29:42.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 24 Sep 2026 06:42:32 UTC on ccycloud.quasar-lgarko.root.comops.site
07:29:42.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4669406
07:29:42.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:29:42.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:42.005041 9059 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:29:42.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:42.006037 9059 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:29:42.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:42.027285 9066 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:29:43.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:42.027673 9065 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:29:43.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:42.028126 9068 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:29:43.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:42.029273 9059 server_base.cc:1034] running on GCE node
07:29:43.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:43.276939 9059 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:29:43.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:43.280157 9059 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:29:43.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:43.281627 9059 hybrid_clock.cc:648] HybridClock initialized: now 1790234983281577 us; error 69 us; skew 500 ppm
07:29:43.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:43.283022 9059 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:29:43.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:29:43 dist-test-slave-hp5j krb5kdc[9046](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790234983, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.210.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:29:43.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:43.288861 9059 init.cc:377] Logged in from keytab as kudu/127.8.210.190@KRBTEST.COM (short username kudu)
07:29:43.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:43.294365 9059 webserver.cc:460] Webserver started at http://127.8.210.190:34947/ using document root <none> and password file <none>
07:29:43.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:43.295305 9059 fs_manager.cc:359] Metadata directory not provided
07:29:43.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:43.295538 9059 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:29:43.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:43.296028 9059 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:29:43.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:43.300792 9059 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-0/data/instance:
07:29:43.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3bea48ecd7654efdabe998ee44e194b3"
07:29:43.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:29:43 on dist-test-slave-hp5j"
07:29:43.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:43.302058 9059 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-0/wal/instance:
07:29:43.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3bea48ecd7654efdabe998ee44e194b3"
07:29:43.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:29:43 on dist-test-slave-hp5j"
07:29:43.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:43.311053 9059 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.008s sys 0.000s
07:29:43.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:43.317395 9077 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:29:43.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:43.319710 9059 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.002s sys 0.002s
07:29:43.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:43.320129 9059 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-0/data,/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-0/wal
07:29:43.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3bea48ecd7654efdabe998ee44e194b3"
07:29:43.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:29:43 on dist-test-slave-hp5j"
07:29:43.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:43.320551 9059 fs_report.cc:383] FS layout report
07:29:43.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:29:43.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-0/wal
07:29:43.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-0/wal
07:29:43.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-0/data/data
07:29:43.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:29:43.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:29:43.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:29:43.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:29:43.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:29:43.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:29:43.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:29:43.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:29:43.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:29:43.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:29:43.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:29:43.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:43.390232 9059 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:29:43.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:43.396873 9059 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:29:43.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:43.397310 9059 kserver.cc:163] Server-wide thread pool size limit: 3276
07:29:43.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:43.484583 9059 rpc_server.cc:307] RPC server started. Bound to: 127.8.210.190:36273
07:29:43.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:43.484655 9128 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.8.210.190:36273 every 8 connection(s)
07:29:43.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:43.487803 9059 server_base.cc:1166] Dumped server information to /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-0/data/info.pb
07:29:43.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:43.493276 9034 external_mini_cluster.cc:1388] Started /tmp/dist-test-task8eqvEo/build/tsan/bin/kudu as pid 9059
07:29:43.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:43.494208 9034 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-0/wal/instance
07:29:43.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:43.494611 9129 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:29:43.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:43.510418 9129 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.210.190" port: 36273 } has no permanent_uuid. Determining permanent_uuid...
07:29:43.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.8.210.189@KRBTEST.COM; defaulting to no policy
07:29:43.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
07:29:43.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.8.210.189@KRBTEST.COM" created.
07:29:43.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:29:43 dist-test-slave-hp5j krb5kdc[9046](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790234983, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.210.190@KRBTEST.COM for kudu/127.8.210.190@KRBTEST.COM
07:29:43.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
07:29:43.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.8.210.189 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/krb5kdc/kudu_127.8.210.189.keytab.
07:29:43.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.8.210.189 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/krb5kdc/kudu_127.8.210.189.keytab.
07:29:43.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:43.550021 9034 external_mini_cluster.cc:1326] Running /tmp/dist-test-task8eqvEo/build/tsan/bin/kudu
07:29:43.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task8eqvEo/build/tsan/bin/kudu
07:29:43.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-1/wal
07:29:43.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-1/data
07:29:43.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:29:43.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:29:43.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:29:43.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:29:43.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:29:43.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:29:43.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-1/logs
07:29:43.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-1/data/info.pb
07:29:43.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:29:43.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:29:43.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:29:43.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:29:43.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:29:43.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:29:43.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:29:43.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:29:43.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:29:43.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:29:43.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:29:43.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:29:43.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.210.189:45987
07:29:43.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.210.189
07:29:43.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:29:43.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/krb5kdc/kudu_127.8.210.189.keytab
07:29:43.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.210.189
07:29:43.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:29:43.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
07:29:43.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
07:29:43.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.210.148:43435
07:29:43.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:29:43.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:29:43.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:29:43.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:29:43.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.210.190:36273,127.8.210.189:45987,127.8.210.188:35193
07:29:43.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:29:43.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44433/jwks.json
07:29:43.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
07:29:43.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:43.561311 9129 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.210.189" port: 45987 } has no permanent_uuid. Determining permanent_uuid...
07:29:43.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:43.563982 9079 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.210.189:45987: connect: Connection refused (error 111)
07:29:43.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:43.568509 9129 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.210.189:45987: Network error: Client connection negotiation failed: client connection to 127.8.210.189:45987: connect: Connection refused (error 111)
07:29:43.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:43.618108 9129 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.210.189" port: 45987 } attempt: 1
07:29:43.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:43.622933 9129 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.210.189:45987: Network error: Client connection negotiation failed: client connection to 127.8.210.189:45987: connect: Connection refused (error 111)
07:29:43.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:43.691382 9129 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.210.189" port: 45987 } attempt: 2
07:29:43.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:43.695334 9129 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.210.189:45987: Network error: Client connection negotiation failed: client connection to 127.8.210.189:45987: connect: Connection refused (error 111)
07:29:43.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:43.786784 9129 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.210.189" port: 45987 } attempt: 3
07:29:43.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:43.790925 9129 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.210.189:45987: Network error: Client connection negotiation failed: client connection to 127.8.210.189:45987: connect: Connection refused (error 111)
07:29:43.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:43.888347 9135 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:29:43.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:43.888872 9135 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:29:43.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:43.889511 9135 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:29:43.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:43.922076 9135 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
07:29:43.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:43.922308 9135 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:29:43.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:43.922564 9135 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:29:43.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:43.922760 9135 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:29:43.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:43.922945 9135 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:29:43.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:43.923123 9135 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:29:43.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:43.923266 9135 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44433/jwks.json
07:29:43.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:43.934366 9129 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.210.189" port: 45987 } attempt: 4
07:29:43.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:43.938619 9129 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.210.189:45987: Network error: Client connection negotiation failed: client connection to 127.8.210.189:45987: connect: Connection refused (error 111)
07:29:43.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:43.961637 9135 master_runner.cc:386] Master server non-default flags:
07:29:43.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:29:43.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.210.148:43435
07:29:43.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:29:43.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:29:43.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-1/data
07:29:43.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-1/wal
07:29:43.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
07:29:43.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:29:43.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.210.190:36273,127.8.210.189:45987,127.8.210.188:35193
07:29:43.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/krb5kdc/kudu_127.8.210.189.keytab
07:29:43.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.210.189
07:29:43.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:29:43.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:29:43.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:29:43.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.210.189:45987
07:29:43.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:29:43.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:29:43.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:29:43.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44433/jwks.json
07:29:43.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:29:43.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
07:29:43.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
07:29:43.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:29:43.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:29:43.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-1/data/info.pb
07:29:43.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.210.189
07:29:43.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:29:43.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:29:43.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:29:43.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:29:43.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:29:43.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:29:43.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-1/logs
07:29:43.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:29:43.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:29:43.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:29:43.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:29:43.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:29:43.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:29:43.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:29:43.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 24 Sep 2026 06:42:32 UTC on ccycloud.quasar-lgarko.root.comops.site
07:29:43.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4669406
07:29:43.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:29:43.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:43.962895 9135 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:29:43.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:43.964219 9135 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:29:43.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:43.984076 9143 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:29:44.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:44.238104 9129 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.210.189" port: 45987 } attempt: 5
07:29:44.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:44.243947 9129 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.210.189:45987: Network error: Client connection negotiation failed: client connection to 127.8.210.189:45987: connect: Connection refused (error 111)
07:29:44.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:44.791553 9129 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.210.189" port: 45987 } attempt: 6
07:29:44.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:44.800225 9129 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.210.189:45987: Network error: Client connection negotiation failed: client connection to 127.8.210.189:45987: connect: Connection refused (error 111)
07:29:45.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:43.985460 9145 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:29:45.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:43.986788 9135 server_base.cc:1034] running on GCE node
07:29:45.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:43.984879 9142 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:29:45.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:45.265208 9135 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:29:45.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:45.268477 9135 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:29:45.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:45.270005 9135 hybrid_clock.cc:648] HybridClock initialized: now 1790234985269958 us; error 64 us; skew 500 ppm
07:29:45.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:45.270910 9135 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:29:45.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:29:45 dist-test-slave-hp5j krb5kdc[9046](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790234985, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.210.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:29:45.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:45.277282 9135 init.cc:377] Logged in from keytab as kudu/127.8.210.189@KRBTEST.COM (short username kudu)
07:29:45.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:45.283236 9135 webserver.cc:460] Webserver started at http://127.8.210.189:43707/ using document root <none> and password file <none>
07:29:45.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:45.284222 9135 fs_manager.cc:359] Metadata directory not provided
07:29:45.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:45.284458 9135 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:29:45.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:45.285013 9135 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:29:45.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:45.290989 9135 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-1/data/instance:
07:29:45.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "39f843007ffe4e3e9b1c1b482bb3490e"
07:29:45.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:29:45 on dist-test-slave-hp5j"
07:29:45.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:45.292624 9135 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-1/wal/instance:
07:29:45.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "39f843007ffe4e3e9b1c1b482bb3490e"
07:29:45.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:29:45 on dist-test-slave-hp5j"
07:29:45.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:45.301661 9135 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.007s sys 0.000s
07:29:45.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:45.308981 9155 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:29:45.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:45.310477 9135 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.007s sys 0.000s
07:29:45.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:45.310848 9135 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-1/data,/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-1/wal
07:29:45.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "39f843007ffe4e3e9b1c1b482bb3490e"
07:29:45.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:29:45 on dist-test-slave-hp5j"
07:29:45.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:45.311208 9135 fs_report.cc:383] FS layout report
07:29:45.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:29:45.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-1/wal
07:29:45.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-1/wal
07:29:45.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-1/data/data
07:29:45.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:29:45.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:29:45.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:29:45.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:29:45.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:29:45.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:29:45.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:29:45.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:29:45.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:29:45.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:29:45.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:29:45.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:45.355405 9135 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:29:45.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:45.361720 9135 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:29:45.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:45.362144 9135 kserver.cc:163] Server-wide thread pool size limit: 3276
07:29:45.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:45.454205 9135 rpc_server.cc:307] RPC server started. Bound to: 127.8.210.189:45987
07:29:45.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:45.454339 9206 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.8.210.189:45987 every 8 connection(s)
07:29:45.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:45.456946 9135 server_base.cc:1166] Dumped server information to /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-1/data/info.pb
07:29:45.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:45.463572 9207 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:29:45.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:45.465296 9034 external_mini_cluster.cc:1388] Started /tmp/dist-test-task8eqvEo/build/tsan/bin/kudu as pid 9135
07:29:45.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:45.465689 9034 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-1/wal/instance
07:29:45.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:45.477260 9207 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.210.190" port: 36273 } has no permanent_uuid. Determining permanent_uuid...
07:29:45.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.8.210.188@KRBTEST.COM; defaulting to no policy
07:29:45.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
07:29:45.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.8.210.188@KRBTEST.COM" created.
07:29:45.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:29:45 dist-test-slave-hp5j krb5kdc[9046](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790234985, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.210.189@KRBTEST.COM for kudu/127.8.210.190@KRBTEST.COM
07:29:45.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
07:29:45.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.8.210.188 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/krb5kdc/kudu_127.8.210.188.keytab.
07:29:45.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.8.210.188 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/krb5kdc/kudu_127.8.210.188.keytab.
07:29:45.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:45.518582 9034 external_mini_cluster.cc:1326] Running /tmp/dist-test-task8eqvEo/build/tsan/bin/kudu
07:29:45.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task8eqvEo/build/tsan/bin/kudu
07:29:45.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-2/wal
07:29:45.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-2/data
07:29:45.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:29:45.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:29:45.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:29:45.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:29:45.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:29:45.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:29:45.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-2/logs
07:29:45.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-2/data/info.pb
07:29:45.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:29:45.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:29:45.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:29:45.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:29:45.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:29:45.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:29:45.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:29:45.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:29:45.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:29:45.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:29:45.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:29:45.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:29:45.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.210.188:35193
07:29:45.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.210.188
07:29:45.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:29:45.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/krb5kdc/kudu_127.8.210.188.keytab
07:29:45.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.210.188
07:29:45.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:29:45.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
07:29:45.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
07:29:45.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.210.148:43435
07:29:45.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:29:45.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:29:45.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:29:45.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:29:45.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.210.190:36273,127.8.210.189:45987,127.8.210.188:35193
07:29:45.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:29:45.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44433/jwks.json
07:29:45.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
07:29:45.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:45.525923 9207 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.210.189" port: 45987 } has no permanent_uuid. Determining permanent_uuid...
07:29:45.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:29:45 dist-test-slave-hp5j krb5kdc[9046](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790234985, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.210.189@KRBTEST.COM for kudu/127.8.210.189@KRBTEST.COM
07:29:45.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:45.558152 9207 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.210.188" port: 35193 } has no permanent_uuid. Determining permanent_uuid...
07:29:45.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:45.565495 9159 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.210.188:35193: connect: Connection refused (error 111)
07:29:45.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:45.569751 9207 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.210.188:35193: Network error: Client connection negotiation failed: client connection to 127.8.210.188:35193: connect: Connection refused (error 111)
07:29:45.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:45.619407 9207 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.210.188" port: 35193 } attempt: 1
07:29:45.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:45.624171 9207 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.210.188:35193: Network error: Client connection negotiation failed: client connection to 127.8.210.188:35193: connect: Connection refused (error 111)
07:29:45.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:45.692837 9207 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.210.188" port: 35193 } attempt: 2
07:29:45.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:45.698160 9207 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.210.188:35193: Network error: Client connection negotiation failed: client connection to 127.8.210.188:35193: connect: Connection refused (error 111)
07:29:45.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:45.789601 9207 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.210.188" port: 35193 } attempt: 3
07:29:45.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:45.793388 9207 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.210.188:35193: Network error: Client connection negotiation failed: client connection to 127.8.210.188:35193: connect: Connection refused (error 111)
07:29:45.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:45.860744 9129 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.210.189" port: 45987 } attempt: 7
07:29:45.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:45.865162 9213 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:29:45.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:45.866328 9213 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:29:45.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:45.867097 9213 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:29:45.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:29:45 dist-test-slave-hp5j krb5kdc[9046](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790234983, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.210.190@KRBTEST.COM for kudu/127.8.210.189@KRBTEST.COM
07:29:45.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:45.886600 9129 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.210.188" port: 35193 } has no permanent_uuid. Determining permanent_uuid...
07:29:45.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:45.893328 9129 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.210.188:35193: Network error: Client connection negotiation failed: client connection to 127.8.210.188:35193: connect: Connection refused (error 111)
07:29:45.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:45.905190 9213 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
07:29:45.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:45.905486 9213 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:29:45.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:45.905799 9213 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:29:45.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:45.906011 9213 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:29:45.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:45.906200 9213 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:29:45.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:45.906466 9213 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:29:45.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:45.906606 9213 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44433/jwks.json
07:29:45.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:45.938649 9207 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.210.188" port: 35193 } attempt: 4
07:29:45.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:45.942373 9207 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.210.188:35193: Network error: Client connection negotiation failed: client connection to 127.8.210.188:35193: connect: Connection refused (error 111)
07:29:45.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:45.946899 9213 master_runner.cc:386] Master server non-default flags:
07:29:45.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:29:45.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.210.148:43435
07:29:45.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:29:45.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:29:45.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-2/data
07:29:45.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-2/wal
07:29:45.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
07:29:45.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:29:45.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.210.190:36273,127.8.210.189:45987,127.8.210.188:35193
07:29:45.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/krb5kdc/kudu_127.8.210.188.keytab
07:29:45.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.210.188
07:29:45.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:29:45.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:29:45.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:29:45.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.210.188:35193
07:29:45.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:29:45.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:29:45.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:29:45.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44433/jwks.json
07:29:45.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:29:45.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
07:29:45.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
07:29:45.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:29:45.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:29:45.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-2/data/info.pb
07:29:45.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.210.188
07:29:45.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:29:45.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:29:45.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:29:45.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:29:45.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:29:45.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:29:45.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-2/logs
07:29:45.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:29:45.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:29:45.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:29:45.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:29:45.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:29:45.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:29:45.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:29:45.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 24 Sep 2026 06:42:32 UTC on ccycloud.quasar-lgarko.root.comops.site
07:29:45.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4669406
07:29:45.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:29:45.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:45.948002 9213 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:29:45.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:45.948909 9213 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:29:45.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:45.951807 9129 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.210.188" port: 35193 } attempt: 1
07:29:45.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:45.955458 9129 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.210.188:35193: Network error: Client connection negotiation failed: client connection to 127.8.210.188:35193: connect: Connection refused (error 111)
07:29:45.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:45.967892 9225 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:29:46.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:46.036931 9129 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.210.188" port: 35193 } attempt: 2
07:29:46.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:46.042026 9129 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.210.188:35193: Network error: Client connection negotiation failed: client connection to 127.8.210.188:35193: connect: Connection refused (error 111)
07:29:46.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:46.127665 9129 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.210.188" port: 35193 } attempt: 3
07:29:46.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:46.131835 9129 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.210.188:35193: Network error: Client connection negotiation failed: client connection to 127.8.210.188:35193: connect: Connection refused (error 111)
07:29:46.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:46.241791 9207 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.210.188" port: 35193 } attempt: 5
07:29:46.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:46.246074 9207 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.210.188:35193: Network error: Client connection negotiation failed: client connection to 127.8.210.188:35193: connect: Connection refused (error 111)
07:29:46.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:46.272281 9129 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.210.188" port: 35193 } attempt: 4
07:29:46.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:46.277189 9129 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.210.188:35193: Network error: Client connection negotiation failed: client connection to 127.8.210.188:35193: connect: Connection refused (error 111)
07:29:46.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:46.560861 9129 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.210.188" port: 35193 } attempt: 5
07:29:46.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:46.567219 9129 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.210.188:35193: Network error: Client connection negotiation failed: client connection to 127.8.210.188:35193: connect: Connection refused (error 111)
07:29:46.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:46.793653 9207 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.210.188" port: 35193 } attempt: 6
07:29:46.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:46.801894 9207 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.210.188:35193: Network error: Client connection negotiation failed: client connection to 127.8.210.188:35193: connect: Connection refused (error 111)
07:29:47.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:47.119838 9129 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.210.188" port: 35193 } attempt: 6
07:29:47.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:47.128165 9129 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.210.188:35193: Network error: Client connection negotiation failed: client connection to 127.8.210.188:35193: connect: Connection refused (error 111)
07:29:47.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:45.967937 9226 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:29:47.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:45.968401 9228 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:29:47.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:45.970625 9213 server_base.cc:1034] running on GCE node
07:29:47.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:47.264568 9213 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:29:47.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:47.267853 9213 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:29:47.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:47.269289 9213 hybrid_clock.cc:648] HybridClock initialized: now 1790234987269230 us; error 78 us; skew 500 ppm
07:29:47.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:47.270143 9213 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:29:47.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:29:47 dist-test-slave-hp5j krb5kdc[9046](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790234987, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.210.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:29:47.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:47.278098 9213 init.cc:377] Logged in from keytab as kudu/127.8.210.188@KRBTEST.COM (short username kudu)
07:29:47.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:47.284847 9213 webserver.cc:460] Webserver started at http://127.8.210.188:46091/ using document root <none> and password file <none>
07:29:47.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:47.286320 9213 fs_manager.cc:359] Metadata directory not provided
07:29:47.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:47.286574 9213 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:29:47.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:47.287086 9213 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:29:47.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:47.293792 9213 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-2/data/instance:
07:29:47.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6107c62b44c140eba62f248d278c05d2"
07:29:47.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:29:47 on dist-test-slave-hp5j"
07:29:47.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:47.295058 9213 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-2/wal/instance:
07:29:47.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6107c62b44c140eba62f248d278c05d2"
07:29:47.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:29:47 on dist-test-slave-hp5j"
07:29:47.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:47.303573 9213 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.005s sys 0.004s
07:29:47.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:47.309821 9241 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:29:47.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:47.311184 9213 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
07:29:47.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:47.311486 9213 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-2/data,/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-2/wal
07:29:47.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6107c62b44c140eba62f248d278c05d2"
07:29:47.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:29:47 on dist-test-slave-hp5j"
07:29:47.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:47.311796 9213 fs_report.cc:383] FS layout report
07:29:47.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:29:47.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-2/wal
07:29:47.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-2/wal
07:29:47.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-2/data/data
07:29:47.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:29:47.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:29:47.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:29:47.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:29:47.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:29:47.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:29:47.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:29:47.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:29:47.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:29:47.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:29:47.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:29:47.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:47.366315 9213 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:29:47.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:47.372522 9213 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:29:47.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:47.372951 9213 kserver.cc:163] Server-wide thread pool size limit: 3276
07:29:47.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:47.472488 9213 rpc_server.cc:307] RPC server started. Bound to: 127.8.210.188:35193
07:29:47.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:47.472613 9292 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.8.210.188:35193 every 8 connection(s)
07:29:47.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:47.475431 9213 server_base.cc:1166] Dumped server information to /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-2/data/info.pb
07:29:47.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:47.481640 9034 external_mini_cluster.cc:1388] Started /tmp/dist-test-task8eqvEo/build/tsan/bin/kudu as pid 9213
07:29:47.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:47.482127 9034 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-2/wal/instance
07:29:47.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:47.489122 9293 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:29:47.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:47.508356 9293 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.210.190" port: 36273 } has no permanent_uuid. Determining permanent_uuid...
07:29:47.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:29:47 dist-test-slave-hp5j krb5kdc[9046](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790234981, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.8.210.188@KRBTEST.COM
07:29:47.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:29:47 dist-test-slave-hp5j krb5kdc[9046](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790234987, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.210.188@KRBTEST.COM for kudu/127.8.210.190@KRBTEST.COM
07:29:47.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:47.548511 9293 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.210.189" port: 45987 } has no permanent_uuid. Determining permanent_uuid...
07:29:47.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:29:47 dist-test-slave-hp5j krb5kdc[9046](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790234987, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.210.188@KRBTEST.COM for kudu/127.8.210.189@KRBTEST.COM
07:29:47.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:29:47 dist-test-slave-hp5j krb5kdc[9046](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790234981, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.8.210.189@KRBTEST.COM
07:29:47.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:47.578450 9293 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.210.188" port: 35193 } has no permanent_uuid. Determining permanent_uuid...
07:29:47.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:29:47 dist-test-slave-hp5j krb5kdc[9046](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790234987, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.210.188@KRBTEST.COM for kudu/127.8.210.188@KRBTEST.COM
07:29:47.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:29:47 dist-test-slave-hp5j krb5kdc[9046](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790234981, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.8.210.190@KRBTEST.COM
07:29:47.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:47.614579 9293 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2: Bootstrap starting.
07:29:47.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:47.622485 9293 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2: Neither blocks nor log segments found. Creating new log.
07:29:47.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:47.625008 9293 log.cc:824] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2: Log is configured to *not* fsync() on all Append() calls
07:29:47.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:47.630138 9034 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
07:29:47.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:47.631129 9293 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2: No bootstrap required, opened a new log
07:29:47.638 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:29:47.640 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
07:29:47.649 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
07:29:47.651 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
07:29:47.654 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.8.210.190"
port: 36273
}
bound_http_address {
host: "127.8.210.190"
port: 34947
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.8.210.189"
port: 45987
}
bound_http_address {
host: "127.8.210.189"
port: 43707
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.8.210.188"
port: 35193
}
bound_http_address {
host: "127.8.210.188"
port: 46091
}
}
}
07:29:47.654 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
07:29:47.655 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:29:47.655 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
07:29:47.656 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
07:29:47.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:47.660379 9293 raft_consensus.cc:348] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3bea48ecd7654efdabe998ee44e194b3" member_type: VOTER last_known_addr { host: "127.8.210.190" port: 36273 } } peers { permanent_uuid: "39f843007ffe4e3e9b1c1b482bb3490e" member_type: VOTER last_known_addr { host: "127.8.210.189" port: 45987 } } peers { permanent_uuid: "6107c62b44c140eba62f248d278c05d2" member_type: VOTER last_known_addr { host: "127.8.210.188" port: 35193 } }
07:29:47.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:47.661252 9293 raft_consensus.cc:374] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:29:47.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:47.661548 9293 raft_consensus.cc:729] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6107c62b44c140eba62f248d278c05d2, State: Initialized, Role: FOLLOWER
07:29:47.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:47.662523 9293 consensus_queue.cc:260] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3bea48ecd7654efdabe998ee44e194b3" member_type: VOTER last_known_addr { host: "127.8.210.190" port: 36273 } } peers { permanent_uuid: "39f843007ffe4e3e9b1c1b482bb3490e" member_type: VOTER last_known_addr { host: "127.8.210.189" port: 45987 } } peers { permanent_uuid: "6107c62b44c140eba62f248d278c05d2" member_type: VOTER last_known_addr { host: "127.8.210.188" port: 35193 } }
07:29:47.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:47.667201 9304 raft_consensus.cc:482] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:29:47.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:47.667689 9304 raft_consensus.cc:504] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3bea48ecd7654efdabe998ee44e194b3" member_type: VOTER last_known_addr { host: "127.8.210.190" port: 36273 } } peers { permanent_uuid: "39f843007ffe4e3e9b1c1b482bb3490e" member_type: VOTER last_known_addr { host: "127.8.210.189" port: 45987 } } peers { permanent_uuid: "6107c62b44c140eba62f248d278c05d2" member_type: VOTER last_known_addr { host: "127.8.210.188" port: 35193 } }
07:29:47.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:47.669464 9293 sys_catalog.cc:564] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [sys.catalog]: configured and running, proceeding with master startup.
07:29:47.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:47.671693 9304 leader_election.cc:288] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3bea48ecd7654efdabe998ee44e194b3 (127.8.210.190:36273), 39f843007ffe4e3e9b1c1b482bb3490e (127.8.210.189:45987)
07:29:47.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:47.672916 9304 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3bea48ecd7654efdabe998ee44e194b3" member_type: VOTER last_known_addr { host: "127.8.210.190" port: 36273 } } peers { permanent_uuid: "39f843007ffe4e3e9b1c1b482bb3490e" member_type: VOTER last_known_addr { host: "127.8.210.189" port: 45987 } } peers { permanent_uuid: "6107c62b44c140eba62f248d278c05d2" member_type: VOTER last_known_addr { host: "127.8.210.188" port: 35193 } } }
07:29:47.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:47.673434 9304 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [sys.catalog]: This master's current role is: FOLLOWER
07:29:47.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:29:47 dist-test-slave-hp5j krb5kdc[9046](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790234987, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:29:47.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
07:29:47.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:47.710145 9316 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:29:47.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:47.710510 9316 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:29:47.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:47.708621 9213 master_runner.cc:186] Error getting master registration for 127.8.210.190:36273: OK, instance_id { permanent_uuid: "3bea48ecd7654efdabe998ee44e194b3" instance_seqno: 1790234983457943 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
07:29:47.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:47.736729 9213 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:29:47.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:47.761648 9034 mini_kdc.cc:317] Time spent kinit for test-admin: real 0.105s user 0.000s sys 0.014s
07:29:47.762 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:29:47.764 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/krb5kdc/krb5cc
07:29:47.771 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
07:29:47.772 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/krb5kdc/krb5cc
07:29:47.774 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
07:29:47.779 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
07:29:47.781 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
07:29:47.782 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
07:29:47.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:29:47 dist-test-slave-hp5j krb5kdc[9046](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790234987, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:29:47.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
07:29:47.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:47.862345 9207 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.210.188" port: 35193 } attempt: 7
07:29:47.867 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:29:47.867 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/krb5kdc/krb5cc
07:29:47.869 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
07:29:47.869 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.8.210.189:45987
07:29:47.871 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
07:29:47.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:47.872375 9034 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task8eqvEo/build/tsan/bin/kudu with pid 9135
07:29:47.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:47.880168 9247 tablet.cc:2369] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2: Can't schedule compaction. Clean time has not been advanced past its initial value.
07:29:47.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:47.905206 9243 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv got EOF from 127.8.210.189:45987 (error 108)
07:29:47.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:47.906405 9296 negotiation.cc:337] Failed RPC negotiation. Trace:
07:29:47.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0924 07:29:47.865037 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:37404 (local address 127.8.210.188:35193)
07:29:47.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0924 07:29:47.865600 (+ 563us) server_negotiation.cc:206] Beginning negotiation
07:29:47.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0924 07:29:47.865626 (+ 26us) server_negotiation.cc:399] Waiting for connection header
07:29:47.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0924 07:29:47.865741 (+ 115us) server_negotiation.cc:407] Connection header received
07:29:47.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0924 07:29:47.866099 (+ 358us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
07:29:47.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0924 07:29:47.866127 (+ 28us) server_negotiation.cc:461] Received NEGOTIATE request from client
07:29:47.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0924 07:29:47.866354 (+ 227us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
07:29:47.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0924 07:29:47.866545 (+ 191us) server_negotiation.cc:226] Negotiated authn=SASL
07:29:47.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0924 07:29:47.904708 (+ 38163us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:37404: BlockingRecv error: recv got EOF from 127.0.0.1:37404 (error 108)
07:29:47.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":129}
07:29:47.908 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:29:47.909 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.8.210.190:36273
07:29:47.909 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
07:29:47.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:47.909847 9034 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task8eqvEo/build/tsan/bin/kudu with pid 9059
07:29:47.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:47.924482 9243 leader_election.cc:334] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 39f843007ffe4e3e9b1c1b482bb3490e (127.8.210.189:45987): Network error: Client connection negotiation failed: client connection to 127.8.210.189:45987: connect: Connection refused (error 111)
07:29:47.947 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:29:47.947 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.8.210.188:35193
07:29:47.948 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
07:29:47.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:47.948940 9034 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task8eqvEo/build/tsan/bin/kudu with pid 9213
07:29:47.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:47.948820 9244 leader_election.cc:334] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 3bea48ecd7654efdabe998ee44e194b3 (127.8.210.190:36273): Network error: Client connection negotiation failed: client connection to 127.8.210.190:36273: connect: Connection refused (error 111)
07:29:47.975 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:29:47.975 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.8.210.189:45987
07:29:47.977 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
07:29:47.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:47.978612 9034 external_mini_cluster.cc:1326] Running /tmp/dist-test-task8eqvEo/build/tsan/bin/kudu
07:29:47.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task8eqvEo/build/tsan/bin/kudu
07:29:47.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-1/wal
07:29:47.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-1/data
07:29:47.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:29:47.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:29:47.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:29:47.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:29:47.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:29:47.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:29:47.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-1/logs
07:29:47.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-1/data/info.pb
07:29:47.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:29:47.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:29:47.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:29:47.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:29:47.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:29:47.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:29:47.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:29:47.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:29:47.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:29:47.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:29:47.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:29:47.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:29:47.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.210.189:45987
07:29:47.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.210.189
07:29:47.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43707
07:29:47.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/krb5kdc/kudu_127.8.210.189.keytab
07:29:47.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.210.189
07:29:47.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:29:47.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
07:29:47.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
07:29:47.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.210.148:43435
07:29:47.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:29:47.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:29:47.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:29:47.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:29:47.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.210.190:36273,127.8.210.189:45987,127.8.210.188:35193
07:29:47.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:29:47.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44433/jwks.json
07:29:47.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
07:29:48.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:48.306641 9324 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:29:48.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:48.307335 9324 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:29:48.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:48.308071 9324 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:29:48.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:48.341985 9324 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
07:29:48.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:48.342288 9324 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:29:48.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:48.342698 9324 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:29:48.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:48.343031 9324 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:29:48.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:48.343355 9324 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:29:48.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:48.343668 9324 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:29:48.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:48.343899 9324 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44433/jwks.json
07:29:48.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:48.384475 9324 master_runner.cc:386] Master server non-default flags:
07:29:48.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:29:48.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.210.148:43435
07:29:48.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:29:48.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:29:48.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-1/data
07:29:48.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-1/wal
07:29:48.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
07:29:48.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:29:48.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.210.190:36273,127.8.210.189:45987,127.8.210.188:35193
07:29:48.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/krb5kdc/kudu_127.8.210.189.keytab
07:29:48.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.210.189
07:29:48.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:29:48.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:29:48.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:29:48.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.210.189:45987
07:29:48.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:29:48.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:29:48.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:29:48.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44433/jwks.json
07:29:48.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:29:48.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
07:29:48.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
07:29:48.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:29:48.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:29:48.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-1/data/info.pb
07:29:48.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.210.189
07:29:48.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43707
07:29:48.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:29:48.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:29:48.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:29:48.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:29:48.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:29:48.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-1/logs
07:29:48.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:29:48.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:29:48.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:29:48.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:29:48.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:29:48.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:29:48.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:29:48.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 24 Sep 2026 06:42:32 UTC on ccycloud.quasar-lgarko.root.comops.site
07:29:48.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4669406
07:29:48.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:29:48.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:48.386915 9324 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:29:48.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:48.388558 9324 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:29:48.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:48.416966 9330 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:29:49.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:48.416977 9333 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:29:49.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:48.418363 9324 server_base.cc:1034] running on GCE node
07:29:49.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:48.419518 9331 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:29:49.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:49.700664 9324 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:29:49.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:49.703935 9324 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:29:49.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:49.705340 9324 hybrid_clock.cc:648] HybridClock initialized: now 1790234989705297 us; error 71 us; skew 500 ppm
07:29:49.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:49.706185 9324 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:29:49.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:29:49 dist-test-slave-hp5j krb5kdc[9046](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790234989, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.210.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:29:49.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:49.712634 9324 init.cc:377] Logged in from keytab as kudu/127.8.210.189@KRBTEST.COM (short username kudu)
07:29:49.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:49.717760 9324 webserver.cc:460] Webserver started at http://127.8.210.189:43707/ using document root <none> and password file <none>
07:29:49.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:49.718685 9324 fs_manager.cc:359] Metadata directory not provided
07:29:49.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:49.718909 9324 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:29:49.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:49.727149 9324 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.000s sys 0.004s
07:29:49.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:49.731560 9341 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:29:49.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:49.732616 9324 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
07:29:49.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:49.732961 9324 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-1/data,/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-1/wal
07:29:49.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "39f843007ffe4e3e9b1c1b482bb3490e"
07:29:49.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:29:45 on dist-test-slave-hp5j"
07:29:49.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:49.735790 9324 fs_report.cc:383] FS layout report
07:29:49.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:29:49.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-1/wal
07:29:49.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-1/wal
07:29:49.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-1/data/data
07:29:49.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:29:49.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:29:49.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:29:49.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:29:49.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:29:49.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:29:49.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:29:49.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:29:49.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:29:49.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:29:49.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:29:49.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:49.771677 9324 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:29:49.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:49.778946 9324 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:29:49.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:49.779368 9324 kserver.cc:163] Server-wide thread pool size limit: 3276
07:29:49.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:49.851840 9324 rpc_server.cc:307] RPC server started. Bound to: 127.8.210.189:45987
07:29:49.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:49.852025 9393 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.8.210.189:45987 every 8 connection(s)
07:29:49.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:49.855046 9324 server_base.cc:1166] Dumped server information to /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-1/data/info.pb
07:29:49.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:49.859972 9034 external_mini_cluster.cc:1388] Started /tmp/dist-test-task8eqvEo/build/tsan/bin/kudu as pid 9324
07:29:49.860 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:29:49.860 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.8.210.190:36273
07:29:49.861 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
07:29:49.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:49.862839 9034 external_mini_cluster.cc:1326] Running /tmp/dist-test-task8eqvEo/build/tsan/bin/kudu
07:29:49.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task8eqvEo/build/tsan/bin/kudu
07:29:49.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-0/wal
07:29:49.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-0/data
07:29:49.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:29:49.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:29:49.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:29:49.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:29:49.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:29:49.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:29:49.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-0/logs
07:29:49.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-0/data/info.pb
07:29:49.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:29:49.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:29:49.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:29:49.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:29:49.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:29:49.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:29:49.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:29:49.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:29:49.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:29:49.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:29:49.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:29:49.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:29:49.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.210.190:36273
07:29:49.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.210.190
07:29:49.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34947
07:29:49.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/krb5kdc/kudu_127.8.210.190.keytab
07:29:49.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.210.190
07:29:49.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:29:49.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
07:29:49.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
07:29:49.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.210.148:43435
07:29:49.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:29:49.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:29:49.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:29:49.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:29:49.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.210.190:36273,127.8.210.189:45987,127.8.210.188:35193
07:29:49.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:29:49.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44433/jwks.json
07:29:49.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
07:29:49.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:49.868695 9394 sys_catalog.cc:263] Verifying existing consensus state
07:29:49.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260924 07:29:49.869556 9394 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
07:29:49.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:49.869954 9324 master.cc:537] Master@127.8.210.189:45987 shutting down...
07:29:49.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:49.927598 9324 master.cc:552] Master@127.8.210.189:45987 shutdown complete.
07:29:49.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
07:29:50.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:50.192837 9396 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:29:50.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:50.193625 9396 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:29:50.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:50.194355 9396 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:29:50.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:50.225807 9396 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
07:29:50.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:50.226097 9396 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:29:50.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:50.226511 9396 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:29:50.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:50.226832 9396 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:29:50.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:50.227118 9396 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:29:50.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:50.227411 9396 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:29:50.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:50.227640 9396 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44433/jwks.json
07:29:50.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:50.265679 9396 master_runner.cc:386] Master server non-default flags:
07:29:50.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:29:50.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.210.148:43435
07:29:50.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:29:50.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:29:50.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-0/data
07:29:50.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-0/wal
07:29:50.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
07:29:50.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:29:50.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.210.190:36273,127.8.210.189:45987,127.8.210.188:35193
07:29:50.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/krb5kdc/kudu_127.8.210.190.keytab
07:29:50.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.210.190
07:29:50.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:29:50.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:29:50.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:29:50.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.210.190:36273
07:29:50.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:29:50.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:29:50.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:29:50.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44433/jwks.json
07:29:50.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:29:50.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
07:29:50.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
07:29:50.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:29:50.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:29:50.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-0/data/info.pb
07:29:50.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.210.190
07:29:50.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34947
07:29:50.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:29:50.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:29:50.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:29:50.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:29:50.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:29:50.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-0/logs
07:29:50.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:29:50.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:29:50.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:29:50.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:29:50.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:29:50.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:29:50.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:29:50.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 24 Sep 2026 06:42:32 UTC on ccycloud.quasar-lgarko.root.comops.site
07:29:50.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4669406
07:29:50.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:29:50.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:50.267204 9396 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:29:50.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:50.268534 9396 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:29:50.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:50.287278 9404 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:29:51.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:50.287590 9406 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:29:51.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:50.287429 9403 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:29:51.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:51.603843 9405 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1314 milliseconds
07:29:51.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:51.603979 9396 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
07:29:51.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:51.605611 9396 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:29:51.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:51.608621 9396 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:29:51.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:51.610213 9396 hybrid_clock.cc:648] HybridClock initialized: now 1790234991610166 us; error 36 us; skew 500 ppm
07:29:51.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:51.612110 9396 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:29:51.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:29:51 dist-test-slave-hp5j krb5kdc[9046](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790234991, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.210.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:29:51.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:51.625469 9396 init.cc:377] Logged in from keytab as kudu/127.8.210.190@KRBTEST.COM (short username kudu)
07:29:51.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:51.631623 9396 webserver.cc:460] Webserver started at http://127.8.210.190:34947/ using document root <none> and password file <none>
07:29:51.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:51.632650 9396 fs_manager.cc:359] Metadata directory not provided
07:29:51.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:51.632925 9396 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:29:51.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:51.640578 9396 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.003s sys 0.003s
07:29:51.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:51.644757 9414 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:29:51.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:51.646023 9396 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
07:29:51.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:51.646409 9396 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-0/data,/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-0/wal
07:29:51.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3bea48ecd7654efdabe998ee44e194b3"
07:29:51.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:29:43 on dist-test-slave-hp5j"
07:29:51.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:51.649549 9396 fs_report.cc:383] FS layout report
07:29:51.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:29:51.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-0/wal
07:29:51.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-0/wal
07:29:51.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-0/data/data
07:29:51.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:29:51.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:29:51.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:29:51.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:29:51.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:29:51.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:29:51.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:29:51.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:29:51.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:29:51.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:29:51.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:29:51.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:51.700502 9396 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:29:51.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:51.706398 9396 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:29:51.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:51.706935 9396 kserver.cc:163] Server-wide thread pool size limit: 3276
07:29:51.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:51.779029 9396 rpc_server.cc:307] RPC server started. Bound to: 127.8.210.190:36273
07:29:51.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:51.779158 9465 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.8.210.190:36273 every 8 connection(s)
07:29:51.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:51.782066 9396 server_base.cc:1166] Dumped server information to /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-0/data/info.pb
07:29:51.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:51.787657 9034 external_mini_cluster.cc:1388] Started /tmp/dist-test-task8eqvEo/build/tsan/bin/kudu as pid 9396
07:29:51.789 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:29:51.789 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.8.210.188:35193
07:29:51.789 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
07:29:51.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:51.790652 9034 external_mini_cluster.cc:1326] Running /tmp/dist-test-task8eqvEo/build/tsan/bin/kudu
07:29:51.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task8eqvEo/build/tsan/bin/kudu
07:29:51.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-2/wal
07:29:51.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-2/data
07:29:51.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:29:51.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:29:51.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:29:51.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:29:51.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:29:51.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:29:51.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-2/logs
07:29:51.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-2/data/info.pb
07:29:51.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:29:51.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:29:51.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:29:51.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:29:51.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:29:51.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:29:51.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:29:51.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:29:51.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:29:51.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:29:51.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:29:51.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:29:51.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.210.188:35193
07:29:51.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.210.188
07:29:51.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46091
07:29:51.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/krb5kdc/kudu_127.8.210.188.keytab
07:29:51.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.210.188
07:29:51.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:29:51.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
07:29:51.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
07:29:51.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.210.148:43435
07:29:51.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:29:51.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:29:51.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:29:51.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:29:51.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.210.190:36273,127.8.210.189:45987,127.8.210.188:35193
07:29:51.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:29:51.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44433/jwks.json
07:29:51.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
07:29:51.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:51.793926 9466 sys_catalog.cc:263] Verifying existing consensus state
07:29:51.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260924 07:29:51.794684 9466 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
07:29:51.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:51.795109 9396 master.cc:537] Master@127.8.210.190:36273 shutting down...
07:29:51.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:51.854600 9396 master.cc:552] Master@127.8.210.190:36273 shutdown complete.
07:29:51.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
07:29:52.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:52.108624 9468 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:29:52.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:52.109166 9468 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:29:52.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:52.109827 9468 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:29:52.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:52.142155 9468 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
07:29:52.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:52.142409 9468 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:29:52.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:52.142664 9468 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:29:52.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:52.142861 9468 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:29:52.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:52.143035 9468 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:29:52.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:52.143206 9468 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:29:52.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:52.143359 9468 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44433/jwks.json
07:29:52.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:52.181397 9468 master_runner.cc:386] Master server non-default flags:
07:29:52.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:29:52.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.210.148:43435
07:29:52.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:29:52.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:29:52.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-2/data
07:29:52.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-2/wal
07:29:52.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
07:29:52.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:29:52.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.210.190:36273,127.8.210.189:45987,127.8.210.188:35193
07:29:52.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/krb5kdc/kudu_127.8.210.188.keytab
07:29:52.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.210.188
07:29:52.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:29:52.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:29:52.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:29:52.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.210.188:35193
07:29:52.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:29:52.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:29:52.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:29:52.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44433/jwks.json
07:29:52.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:29:52.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
07:29:52.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
07:29:52.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:29:52.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:29:52.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-2/data/info.pb
07:29:52.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.210.188
07:29:52.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46091
07:29:52.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:29:52.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:29:52.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:29:52.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:29:52.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:29:52.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-2/logs
07:29:52.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:29:52.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:29:52.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:29:52.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:29:52.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:29:52.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:29:52.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:29:52.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 24 Sep 2026 06:42:32 UTC on ccycloud.quasar-lgarko.root.comops.site
07:29:52.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4669406
07:29:52.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:29:52.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:52.182622 9468 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:29:52.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:52.183638 9468 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:29:52.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:52.201309 9475 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:29:53.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:52.201586 9477 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:29:53.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:52.201815 9474 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:29:53.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:52.204152 9468 server_base.cc:1034] running on GCE node
07:29:53.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:53.453050 9468 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:29:53.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:53.455816 9468 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:29:53.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:53.457276 9468 hybrid_clock.cc:648] HybridClock initialized: now 1790234993457236 us; error 59 us; skew 500 ppm
07:29:53.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:53.458343 9468 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:29:53.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:29:53 dist-test-slave-hp5j krb5kdc[9046](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790234993, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.210.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:29:53.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:53.464905 9468 init.cc:377] Logged in from keytab as kudu/127.8.210.188@KRBTEST.COM (short username kudu)
07:29:53.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:53.470151 9468 webserver.cc:460] Webserver started at http://127.8.210.188:46091/ using document root <none> and password file <none>
07:29:53.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:53.471024 9468 fs_manager.cc:359] Metadata directory not provided
07:29:53.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:53.471218 9468 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:29:53.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:53.480575 9468 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.005s sys 0.000s
07:29:53.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:53.485813 9486 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:29:53.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:53.487280 9468 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.001s sys 0.003s
07:29:53.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:53.487640 9468 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-2/data,/tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-2/wal
07:29:53.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6107c62b44c140eba62f248d278c05d2"
07:29:53.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:29:47 on dist-test-slave-hp5j"
07:29:53.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:53.490438 9468 fs_report.cc:383] FS layout report
07:29:53.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:29:53.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-2/wal
07:29:53.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-2/wal
07:29:53.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-2/data/data
07:29:53.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:29:53.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:29:53.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:29:53.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:29:53.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:29:53.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:29:53.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:29:53.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:29:53.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:29:53.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:29:53.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:29:53.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:53.535790 9468 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:29:53.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:53.541719 9468 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:29:53.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:53.542227 9468 kserver.cc:163] Server-wide thread pool size limit: 3276
07:29:53.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:53.631760 9468 rpc_server.cc:307] RPC server started. Bound to: 127.8.210.188:35193
07:29:53.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:53.632402 9538 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.8.210.188:35193 every 8 connection(s)
07:29:53.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:53.635756 9468 server_base.cc:1166] Dumped server information to /tmp/dist-test-task8eqvEo/test-tmp/mini-kudu-cluster6847391696049371236/master-2/data/info.pb
07:29:53.640 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:29:53.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:53.638597 9034 external_mini_cluster.cc:1388] Started /tmp/dist-test-task8eqvEo/build/tsan/bin/kudu as pid 9468
07:29:53.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:53.647920 9540 sys_catalog.cc:263] Verifying existing consensus state
07:29:53.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:53.654699 9540 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2: Bootstrap starting.
07:29:53.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:53.697510 9540 log.cc:824] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2: Log is configured to *not* fsync() on all Append() calls
07:29:53.699 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.8.210.189:45987(127.8.210.189:45987)] connecting to peer
07:29:53.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:53.703088 9540 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
07:29:53.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:53.703985 9540 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2: Bootstrap complete.
07:29:53.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:53.730947 9540 raft_consensus.cc:348] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3bea48ecd7654efdabe998ee44e194b3" member_type: VOTER last_known_addr { host: "127.8.210.190" port: 36273 } } peers { permanent_uuid: "39f843007ffe4e3e9b1c1b482bb3490e" member_type: VOTER last_known_addr { host: "127.8.210.189" port: 45987 } } peers { permanent_uuid: "6107c62b44c140eba62f248d278c05d2" member_type: VOTER last_known_addr { host: "127.8.210.188" port: 35193 } }
07:29:53.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:53.733618 9540 raft_consensus.cc:374] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:29:53.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:53.733944 9540 raft_consensus.cc:729] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6107c62b44c140eba62f248d278c05d2, State: Initialized, Role: FOLLOWER
07:29:53.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:53.734891 9540 consensus_queue.cc:260] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3bea48ecd7654efdabe998ee44e194b3" member_type: VOTER last_known_addr { host: "127.8.210.190" port: 36273 } } peers { permanent_uuid: "39f843007ffe4e3e9b1c1b482bb3490e" member_type: VOTER last_known_addr { host: "127.8.210.189" port: 45987 } } peers { permanent_uuid: "6107c62b44c140eba62f248d278c05d2" member_type: VOTER last_known_addr { host: "127.8.210.188" port: 35193 } }
07:29:53.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:53.746534 9544 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3bea48ecd7654efdabe998ee44e194b3" member_type: VOTER last_known_addr { host: "127.8.210.190" port: 36273 } } peers { permanent_uuid: "39f843007ffe4e3e9b1c1b482bb3490e" member_type: VOTER last_known_addr { host: "127.8.210.189" port: 45987 } } peers { permanent_uuid: "6107c62b44c140eba62f248d278c05d2" member_type: VOTER last_known_addr { host: "127.8.210.188" port: 35193 } } }
07:29:53.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:53.747426 9544 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [sys.catalog]: This master's current role is: FOLLOWER
07:29:53.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:53.747887 9540 sys_catalog.cc:564] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [sys.catalog]: configured and running, proceeding with master startup.
07:29:53.782 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.8.210.190:36273(127.8.210.190:36273)] connecting to peer
07:29:53.784 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.8.210.188:35193(127.8.210.188:35193)] connecting to peer
07:29:53.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:53.792596 9555 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:29:53.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:53.793017 9555 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:29:53.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:53.798429 9492 tablet.cc:2369] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2: Can't schedule compaction. Clean time has not been advanced past its initial value.
07:29:53.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:53.795492 9468 master_runner.cc:186] Error getting master registration for 127.8.210.190:36273: Network error: Client connection negotiation failed: client connection to 127.8.210.190:36273: connect: Connection refused (error 111),
07:29:53.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:53.827142 9468 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:29:53.844 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:29:53.845 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:29:53.846 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.210.190:36273(127.8.210.190:36273)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:29:53.854 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.210.190:36273: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:29:53.855 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.210.189:45987(127.8.210.189:45987)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:29:53.856 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.210.189:45987: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:29:53.857 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.8.210.188:35193(127.8.210.188:35193)] Successfully connected to peer
07:29:53.939 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
07:29:53.949 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.8.210.188
07:29:54.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:54.149617 9544 raft_consensus.cc:482] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:29:54.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:54.150218 9544 raft_consensus.cc:504] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3bea48ecd7654efdabe998ee44e194b3" member_type: VOTER last_known_addr { host: "127.8.210.190" port: 36273 } } peers { permanent_uuid: "39f843007ffe4e3e9b1c1b482bb3490e" member_type: VOTER last_known_addr { host: "127.8.210.189" port: 45987 } } peers { permanent_uuid: "6107c62b44c140eba62f248d278c05d2" member_type: VOTER last_known_addr { host: "127.8.210.188" port: 35193 } }
07:29:54.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:54.155045 9544 leader_election.cc:288] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3bea48ecd7654efdabe998ee44e194b3 (127.8.210.190:36273), 39f843007ffe4e3e9b1c1b482bb3490e (127.8.210.189:45987)
07:29:54.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:54.155723 9489 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.210.190:36273: connect: Connection refused (error 111)
07:29:54.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:29:54 dist-test-slave-hp5j krb5kdc[9046](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790234987, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.8.210.188@KRBTEST.COM
07:29:54.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:54.172629 9489 leader_election.cc:334] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 3bea48ecd7654efdabe998ee44e194b3 (127.8.210.190:36273): Network error: Client connection negotiation failed: client connection to 127.8.210.190:36273: connect: Connection refused (error 111)
07:29:54.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:54.180270 9488 leader_election.cc:334] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 39f843007ffe4e3e9b1c1b482bb3490e (127.8.210.189:45987): Network error: Client connection negotiation failed: client connection to 127.8.210.189:45987: connect: Connection refused (error 111)
07:29:54.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:54.180708 9488 leader_election.cc:302] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6107c62b44c140eba62f248d278c05d2; no voters: 39f843007ffe4e3e9b1c1b482bb3490e, 3bea48ecd7654efdabe998ee44e194b3
07:29:54.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:54.181603 9544 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
07:29:54.208 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xe03b0512, L:/127.0.0.1:52476 - R:/127.8.210.188:35193] using SASL/GSSAPI
07:29:54.229 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193; will retry
07:29:54.246 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.210.189:45987(127.8.210.189:45987)] connecting to peer
07:29:54.250 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.210.190:36273(127.8.210.190:36273)] connecting to peer
07:29:54.251 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:29:54.251 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.210.189:45987(127.8.210.189:45987)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:29:54.252 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.210.189:45987: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:29:54.262 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:29:54.262 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.210.190:36273(127.8.210.190:36273)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:29:54.262 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.210.190:36273: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:29:54.265 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193; will retry
07:29:54.285 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.210.189:45987(127.8.210.189:45987)] connecting to peer
07:29:54.286 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.210.190:36273(127.8.210.190:36273)] connecting to peer
07:29:54.288 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:29:54.288 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.210.189:45987(127.8.210.189:45987)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:29:54.288 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.210.189:45987: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:29:54.290 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:29:54.290 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.210.190:36273(127.8.210.190:36273)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:29:54.290 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.210.190:36273: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:29:54.293 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193; will retry
07:29:54.304 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.210.189:45987(127.8.210.189:45987)] connecting to peer
07:29:54.305 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.210.190:36273(127.8.210.190:36273)] connecting to peer
07:29:54.307 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:29:54.307 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.210.189:45987(127.8.210.189:45987)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:29:54.308 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.210.189:45987: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:29:54.311 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:29:54.311 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.210.190:36273(127.8.210.190:36273)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:29:54.311 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.210.190:36273: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:29:54.313 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193; will retry
07:29:54.324 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.210.189:45987(127.8.210.189:45987)] connecting to peer
07:29:54.325 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.210.190:36273(127.8.210.190:36273)] connecting to peer
07:29:54.327 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:29:54.327 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.210.189:45987(127.8.210.189:45987)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:29:54.329 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.210.189:45987: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:29:54.331 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:29:54.331 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.210.190:36273(127.8.210.190:36273)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:29:54.332 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.210.190:36273: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:29:54.334 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193; will retry
07:29:54.364 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.210.189:45987(127.8.210.189:45987)] connecting to peer
07:29:54.366 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.210.190:36273(127.8.210.190:36273)] connecting to peer
07:29:54.367 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:29:54.367 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.210.189:45987(127.8.210.189:45987)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:29:54.367 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.210.189:45987: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:29:54.377 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:29:54.377 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.210.190:36273(127.8.210.190:36273)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:29:54.378 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.210.190:36273: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:29:54.378 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193; will retry
07:29:54.445 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.210.189:45987(127.8.210.189:45987)] connecting to peer
07:29:54.447 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.210.190:36273(127.8.210.190:36273)] connecting to peer
07:29:54.447 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:29:54.448 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.210.189:45987(127.8.210.189:45987)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:29:54.449 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.210.189:45987: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:29:54.452 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:29:54.452 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.210.190:36273(127.8.210.190:36273)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:29:54.453 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.210.190:36273: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:29:54.455 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193; will retry
07:29:54.485 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.210.189:45987(127.8.210.189:45987)] connecting to peer
07:29:54.487 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.210.190:36273(127.8.210.190:36273)] connecting to peer
07:29:54.487 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:29:54.488 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.210.189:45987(127.8.210.189:45987)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:29:54.489 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.210.189:45987: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:29:54.490 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:29:54.490 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.210.190:36273(127.8.210.190:36273)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:29:54.492 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.210.190:36273: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:29:54.493 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193; will retry
07:29:54.645 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.210.189:45987(127.8.210.189:45987)] connecting to peer
07:29:54.646 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.210.190:36273(127.8.210.190:36273)] connecting to peer
07:29:54.648 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:29:54.648 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.210.189:45987(127.8.210.189:45987)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:29:54.650 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.210.189:45987: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:29:54.652 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:29:54.652 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.210.190:36273(127.8.210.190:36273)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:29:54.652 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.210.190:36273: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:29:54.654 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193; will retry
07:29:54.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:54.797166 9555 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:29:54.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:54.797537 9555 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:29:54.924 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.210.189:45987(127.8.210.189:45987)] connecting to peer
07:29:54.926 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.210.190:36273(127.8.210.190:36273)] connecting to peer
07:29:54.926 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:29:54.927 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.210.189:45987(127.8.210.189:45987)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:29:54.927 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.210.189:45987: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:29:54.928 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:29:54.928 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.210.190:36273(127.8.210.190:36273)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:29:54.929 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.210.190:36273: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:29:54.933 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193; will retry
07:29:55.465 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.210.189:45987(127.8.210.189:45987)] connecting to peer
07:29:55.466 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.210.190:36273(127.8.210.190:36273)] connecting to peer
07:29:55.467 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:29:55.467 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.210.189:45987(127.8.210.189:45987)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:29:55.467 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.210.189:45987: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:29:55.469 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:29:55.469 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.210.190:36273(127.8.210.190:36273)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:29:55.470 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.210.190:36273: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:29:55.472 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193; will retry
07:29:55.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:55.800469 9555 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:29:55.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:55.800719 9555 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:29:56.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:56.074761 9565 raft_consensus.cc:482] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:29:56.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:56.075099 9565 raft_consensus.cc:504] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3bea48ecd7654efdabe998ee44e194b3" member_type: VOTER last_known_addr { host: "127.8.210.190" port: 36273 } } peers { permanent_uuid: "39f843007ffe4e3e9b1c1b482bb3490e" member_type: VOTER last_known_addr { host: "127.8.210.189" port: 45987 } } peers { permanent_uuid: "6107c62b44c140eba62f248d278c05d2" member_type: VOTER last_known_addr { host: "127.8.210.188" port: 35193 } }
07:29:56.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:56.077247 9565 leader_election.cc:288] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3bea48ecd7654efdabe998ee44e194b3 (127.8.210.190:36273), 39f843007ffe4e3e9b1c1b482bb3490e (127.8.210.189:45987)
07:29:56.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:56.085101 9489 leader_election.cc:334] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 3bea48ecd7654efdabe998ee44e194b3 (127.8.210.190:36273): Network error: Client connection negotiation failed: client connection to 127.8.210.190:36273: connect: Connection refused (error 111)
07:29:56.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:56.085601 9488 leader_election.cc:334] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 39f843007ffe4e3e9b1c1b482bb3490e (127.8.210.189:45987): Network error: Client connection negotiation failed: client connection to 127.8.210.189:45987: connect: Connection refused (error 111)
07:29:56.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:56.085918 9488 leader_election.cc:302] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6107c62b44c140eba62f248d278c05d2; no voters: 39f843007ffe4e3e9b1c1b482bb3490e, 3bea48ecd7654efdabe998ee44e194b3
07:29:56.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:56.086490 9565 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
07:29:56.105 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.210.189:45987(127.8.210.189:45987)] connecting to peer
07:29:56.106 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.210.190:36273(127.8.210.190:36273)] connecting to peer
07:29:56.108 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:29:56.108 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.210.189:45987(127.8.210.189:45987)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:29:56.108 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.210.189:45987: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:29:56.109 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:29:56.109 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.210.190:36273(127.8.210.190:36273)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:29:56.109 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.210.190:36273: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:29:56.111 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193; will retry
07:29:56.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:56.803579 9555 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:29:56.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:56.803822 9555 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:29:57.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:57.806635 9555 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:29:57.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:57.806947 9555 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:29:58.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:58.298581 9572 raft_consensus.cc:482] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:29:58.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:58.298947 9572 raft_consensus.cc:504] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3bea48ecd7654efdabe998ee44e194b3" member_type: VOTER last_known_addr { host: "127.8.210.190" port: 36273 } } peers { permanent_uuid: "39f843007ffe4e3e9b1c1b482bb3490e" member_type: VOTER last_known_addr { host: "127.8.210.189" port: 45987 } } peers { permanent_uuid: "6107c62b44c140eba62f248d278c05d2" member_type: VOTER last_known_addr { host: "127.8.210.188" port: 35193 } }
07:29:58.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:58.300436 9572 leader_election.cc:288] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3bea48ecd7654efdabe998ee44e194b3 (127.8.210.190:36273), 39f843007ffe4e3e9b1c1b482bb3490e (127.8.210.189:45987)
07:29:58.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:58.305522 9489 leader_election.cc:334] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 3bea48ecd7654efdabe998ee44e194b3 (127.8.210.190:36273): Network error: Client connection negotiation failed: client connection to 127.8.210.190:36273: connect: Connection refused (error 111)
07:29:58.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:58.305951 9488 leader_election.cc:334] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 39f843007ffe4e3e9b1c1b482bb3490e (127.8.210.189:45987): Network error: Client connection negotiation failed: client connection to 127.8.210.189:45987: connect: Connection refused (error 111)
07:29:58.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:58.306258 9488 leader_election.cc:302] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6107c62b44c140eba62f248d278c05d2; no voters: 39f843007ffe4e3e9b1c1b482bb3490e, 3bea48ecd7654efdabe998ee44e194b3
07:29:58.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:58.306831 9572 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
07:29:58.425 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.210.189:45987(127.8.210.189:45987)] connecting to peer
07:29:58.426 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.210.190:36273(127.8.210.190:36273)] connecting to peer
07:29:58.427 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:29:58.427 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.210.189:45987(127.8.210.189:45987)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:29:58.427 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.210.189:45987: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:29:58.429 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:29:58.429 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.210.190:36273(127.8.210.190:36273)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:29:58.429 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.210.190:36273: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:29:58.431 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193; will retry
07:29:58.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:58.810110 9555 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:29:58.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:58.810345 9555 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:29:58.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:58.829466 9468 master_runner.cc:186] Error getting master registration for 127.8.210.190:36273: Network error: Client connection negotiation failed: client connection to 127.8.210.190:36273: connect: Connection refused (error 111),
07:29:58.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:29:58.829809 9468 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:29:59.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:59.813153 9555 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:29:59.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:29:59.813403 9555 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:30:00.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:30:00.538753 9579 raft_consensus.cc:482] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:30:00.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:30:00.539093 9579 raft_consensus.cc:504] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3bea48ecd7654efdabe998ee44e194b3" member_type: VOTER last_known_addr { host: "127.8.210.190" port: 36273 } } peers { permanent_uuid: "39f843007ffe4e3e9b1c1b482bb3490e" member_type: VOTER last_known_addr { host: "127.8.210.189" port: 45987 } } peers { permanent_uuid: "6107c62b44c140eba62f248d278c05d2" member_type: VOTER last_known_addr { host: "127.8.210.188" port: 35193 } }
07:30:00.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:30:00.540517 9579 leader_election.cc:288] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3bea48ecd7654efdabe998ee44e194b3 (127.8.210.190:36273), 39f843007ffe4e3e9b1c1b482bb3490e (127.8.210.189:45987)
07:30:00.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:30:00.542309 9489 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.210.190:36273: connect: Connection refused (error 111) [suppressed 5 similar messages]
07:30:00.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:30:00.544802 9488 leader_election.cc:334] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 39f843007ffe4e3e9b1c1b482bb3490e (127.8.210.189:45987): Network error: Client connection negotiation failed: client connection to 127.8.210.189:45987: connect: Connection refused (error 111)
07:30:00.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:30:00.545272 9489 leader_election.cc:334] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 3bea48ecd7654efdabe998ee44e194b3 (127.8.210.190:36273): Network error: Client connection negotiation failed: client connection to 127.8.210.190:36273: connect: Connection refused (error 111)
07:30:00.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:30:00.545647 9489 leader_election.cc:302] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6107c62b44c140eba62f248d278c05d2; no voters: 39f843007ffe4e3e9b1c1b482bb3490e, 3bea48ecd7654efdabe998ee44e194b3
07:30:00.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:30:00.546240 9579 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
07:30:00.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:30:00.816130 9555 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:30:00.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:30:00.816542 9555 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:30:01.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:30:01.819245 9555 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:30:01.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:30:01.819473 9555 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:30:02.425 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.210.189:45987(127.8.210.189:45987)] connecting to peer
07:30:02.426 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.210.190:36273(127.8.210.190:36273)] connecting to peer
07:30:02.427 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:30:02.427 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.210.189:45987(127.8.210.189:45987)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:30:02.427 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.210.189:45987: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:30:02.428 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:30:02.428 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.210.190:36273(127.8.210.190:36273)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:30:02.428 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.210.190:36273: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:30:02.432 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193; will retry
07:30:02.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:30:02.822721 9555 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:30:02.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:30:02.823261 9555 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:30:03.105 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.210.189:45987(127.8.210.189:45987)] connecting to peer
07:30:03.106 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.210.190:36273(127.8.210.190:36273)] connecting to peer
07:30:03.107 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:30:03.107 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.210.189:45987(127.8.210.189:45987)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:30:03.108 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.210.189:45987: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:30:03.109 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:30:03.109 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.210.190:36273(127.8.210.190:36273)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:30:03.109 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.210.190:36273: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:30:03.111 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193; will retry
07:30:03.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:30:03 dist-test-slave-hp5j krb5kdc[9046](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790235003, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.210.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:30:03.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:30:03.468483 9482 init.cc:295] Successfully reacquired a new kerberos TGT
07:30:03.624 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.210.189:45987(127.8.210.189:45987)] connecting to peer
07:30:03.626 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.210.190:36273(127.8.210.190:36273)] connecting to peer
07:30:03.626 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:30:03.626 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.210.189:45987(127.8.210.189:45987)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:30:03.627 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.210.189:45987: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:30:03.628 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:30:03.629 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.210.190:36273(127.8.210.190:36273)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:30:03.629 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.210.190:36273: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:30:03.631 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193; will retry
07:30:03.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:30:03.826083 9555 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:30:03.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:30:03.826344 9555 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:30:03.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:30:03.832686 9468 master_runner.cc:186] Error getting master registration for 127.8.210.190:36273: Network error: Client connection negotiation failed: client connection to 127.8.210.190:36273: connect: Connection refused (error 111),
07:30:03.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:30:03.833097 9468 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:30:03.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:30:03.944830 9587 raft_consensus.cc:482] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:30:03.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:30:03.945405 9587 raft_consensus.cc:504] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3bea48ecd7654efdabe998ee44e194b3" member_type: VOTER last_known_addr { host: "127.8.210.190" port: 36273 } } peers { permanent_uuid: "39f843007ffe4e3e9b1c1b482bb3490e" member_type: VOTER last_known_addr { host: "127.8.210.189" port: 45987 } } peers { permanent_uuid: "6107c62b44c140eba62f248d278c05d2" member_type: VOTER last_known_addr { host: "127.8.210.188" port: 35193 } }
07:30:03.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:30:03.947081 9587 leader_election.cc:288] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3bea48ecd7654efdabe998ee44e194b3 (127.8.210.190:36273), 39f843007ffe4e3e9b1c1b482bb3490e (127.8.210.189:45987)
07:30:03.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:30:03.951303 9489 leader_election.cc:334] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 3bea48ecd7654efdabe998ee44e194b3 (127.8.210.190:36273): Network error: Client connection negotiation failed: client connection to 127.8.210.190:36273: connect: Connection refused (error 111)
07:30:03.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:30:03.951866 9488 leader_election.cc:334] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 39f843007ffe4e3e9b1c1b482bb3490e (127.8.210.189:45987): Network error: Client connection negotiation failed: client connection to 127.8.210.189:45987: connect: Connection refused (error 111)
07:30:03.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:30:03.952214 9488 leader_election.cc:302] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6107c62b44c140eba62f248d278c05d2; no voters: 39f843007ffe4e3e9b1c1b482bb3490e, 3bea48ecd7654efdabe998ee44e194b3
07:30:03.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:30:03.952869 9587 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
07:30:04.464 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.210.189:45987(127.8.210.189:45987)] connecting to peer
07:30:04.466 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.210.190:36273(127.8.210.190:36273)] connecting to peer
07:30:04.467 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:30:04.467 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.210.189:45987(127.8.210.189:45987)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:30:04.467 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.210.189:45987: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:30:04.468 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:30:04.468 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.210.190:36273(127.8.210.190:36273)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:30:04.468 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.210.190:36273: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:30:04.470 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193; will retry
07:30:04.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:30:04.830170 9555 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:30:04.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:30:04.830499 9555 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:30:05.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:30:05.817848 8455 debug-util.cc:398] Leaking SignalData structure 0x7b08000af6a0 after lost signal to thread 8390
07:30:05.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:30:05.818437 8455 debug-util.cc:398] Leaking SignalData structure 0x7b08000ae300 after lost signal to thread 8458
07:30:05.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:30:05.833978 9555 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:30:05.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:30:05.834641 9555 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:30:06.405 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.210.189:45987(127.8.210.189:45987)] connecting to peer
07:30:06.406 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.210.190:36273(127.8.210.190:36273)] connecting to peer
07:30:06.408 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:30:06.408 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.210.189:45987(127.8.210.189:45987)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:30:06.409 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.210.189:45987: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:30:06.410 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:30:06.410 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.210.190:36273(127.8.210.190:36273)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:30:06.410 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.210.190:36273: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:30:06.412 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193; will retry
07:30:06.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:30:06.837653 9555 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:30:06.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:30:06.837903 9555 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:30:07.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:30:07.440061 9594 raft_consensus.cc:482] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:30:07.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:30:07.440564 9594 raft_consensus.cc:504] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3bea48ecd7654efdabe998ee44e194b3" member_type: VOTER last_known_addr { host: "127.8.210.190" port: 36273 } } peers { permanent_uuid: "39f843007ffe4e3e9b1c1b482bb3490e" member_type: VOTER last_known_addr { host: "127.8.210.189" port: 45987 } } peers { permanent_uuid: "6107c62b44c140eba62f248d278c05d2" member_type: VOTER last_known_addr { host: "127.8.210.188" port: 35193 } }
07:30:07.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:30:07.443296 9594 leader_election.cc:288] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3bea48ecd7654efdabe998ee44e194b3 (127.8.210.190:36273), 39f843007ffe4e3e9b1c1b482bb3490e (127.8.210.189:45987)
07:30:07.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:30:07.445257 9489 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.210.190:36273: connect: Connection refused (error 111) [suppressed 3 similar messages]
07:30:07.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:30:07.452492 9489 leader_election.cc:334] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 3bea48ecd7654efdabe998ee44e194b3 (127.8.210.190:36273): Network error: Client connection negotiation failed: client connection to 127.8.210.190:36273: connect: Connection refused (error 111)
07:30:07.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:30:07.462309 9488 leader_election.cc:334] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 39f843007ffe4e3e9b1c1b482bb3490e (127.8.210.189:45987): Network error: Client connection negotiation failed: client connection to 127.8.210.189:45987: connect: Connection refused (error 111)
07:30:07.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:30:07.462999 9488 leader_election.cc:302] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6107c62b44c140eba62f248d278c05d2; no voters: 39f843007ffe4e3e9b1c1b482bb3490e, 3bea48ecd7654efdabe998ee44e194b3
07:30:07.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:30:07.463742 9594 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
07:30:07.525 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.210.189:45987(127.8.210.189:45987)] connecting to peer
07:30:07.526 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.210.190:36273(127.8.210.190:36273)] connecting to peer
07:30:07.527 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:30:07.528 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.210.189:45987(127.8.210.189:45987)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:30:07.528 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.210.189:45987: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:30:07.529 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:30:07.529 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.210.190:36273(127.8.210.190:36273)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:30:07.529 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.210.190:36273: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:30:07.531 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193; will retry
07:30:07.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:30:07.840538 9555 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:30:07.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:30:07.840791 9555 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:30:08.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:30:08.486835 9018 debug-util.cc:398] Leaking SignalData structure 0x7b08000af800 after lost signal to thread 8892
07:30:08.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:30:08.488039 9018 debug-util.cc:398] Leaking SignalData structure 0x7b08000af740 after lost signal to thread 9021
07:30:08.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:30:08.835919 9468 master_runner.cc:186] Error getting master registration for 127.8.210.190:36273: Network error: Client connection negotiation failed: client connection to 127.8.210.190:36273: connect: Connection refused (error 111),
07:30:08.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:30:08.836377 9468 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:30:08.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:30:08.843534 9555 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:30:08.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:30:08.843813 9555 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:30:08.944 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.210.189:45987(127.8.210.189:45987)] connecting to peer
07:30:08.946 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.210.190:36273(127.8.210.190:36273)] connecting to peer
07:30:08.947 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:30:08.947 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.210.189:45987(127.8.210.189:45987)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:30:08.948 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.210.189:45987: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:30:08.948 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:30:08.949 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.210.190:36273(127.8.210.190:36273)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:30:08.949 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.210.190:36273: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:30:08.951 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193; will retry
07:30:09.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:30:09.846768 9555 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:30:09.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:30:09.847051 9555 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:30:10.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:30:10.850001 9555 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:30:10.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:30:10.850286 9555 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:30:11.625 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.210.189:45987(127.8.210.189:45987)] connecting to peer
07:30:11.626 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.210.190:36273(127.8.210.190:36273)] connecting to peer
07:30:11.627 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:30:11.627 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.210.189:45987(127.8.210.189:45987)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:30:11.627 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.210.189:45987: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:30:11.628 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:30:11.628 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.210.190:36273(127.8.210.190:36273)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:30:11.629 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.210.190:36273: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:30:11.632 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193; will retry
07:30:11.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:30:11.853106 9555 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:30:11.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:30:11.853385 9555 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:30:11.885 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.210.189:45987(127.8.210.189:45987)] connecting to peer
07:30:11.886 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.210.190:36273(127.8.210.190:36273)] connecting to peer
07:30:11.886 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:30:11.887 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.210.189:45987(127.8.210.189:45987)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:30:11.887 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.210.189:45987: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:30:11.888 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:30:11.888 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.210.190:36273(127.8.210.190:36273)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:30:11.888 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.210.190:36273: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:30:11.891 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193; will retry
07:30:12.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:30:12.856741 9555 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:30:12.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:30:12.857062 9555 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:30:13.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:30:13 dist-test-slave-hp5j krb5kdc[9046](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790235013, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.210.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:30:13.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:30:13.470877 9482 init.cc:295] Successfully reacquired a new kerberos TGT
07:30:13.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:30:13.478104 9605 raft_consensus.cc:482] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:30:13.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:30:13.478600 9605 raft_consensus.cc:504] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3bea48ecd7654efdabe998ee44e194b3" member_type: VOTER last_known_addr { host: "127.8.210.190" port: 36273 } } peers { permanent_uuid: "39f843007ffe4e3e9b1c1b482bb3490e" member_type: VOTER last_known_addr { host: "127.8.210.189" port: 45987 } } peers { permanent_uuid: "6107c62b44c140eba62f248d278c05d2" member_type: VOTER last_known_addr { host: "127.8.210.188" port: 35193 } }
07:30:13.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:30:13.481196 9605 leader_election.cc:288] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3bea48ecd7654efdabe998ee44e194b3 (127.8.210.190:36273), 39f843007ffe4e3e9b1c1b482bb3490e (127.8.210.189:45987)
07:30:13.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:30:13.484519 9489 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.210.190:36273: connect: Connection refused (error 111) [suppressed 1 similar messages]
07:30:13.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:30:13.493172 9489 leader_election.cc:334] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 3bea48ecd7654efdabe998ee44e194b3 (127.8.210.190:36273): Network error: Client connection negotiation failed: client connection to 127.8.210.190:36273: connect: Connection refused (error 111)
07:30:13.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:30:13.493921 9488 leader_election.cc:334] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 39f843007ffe4e3e9b1c1b482bb3490e (127.8.210.189:45987): Network error: Client connection negotiation failed: client connection to 127.8.210.189:45987: connect: Connection refused (error 111)
07:30:13.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:30:13.494306 9488 leader_election.cc:302] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6107c62b44c140eba62f248d278c05d2; no voters: 39f843007ffe4e3e9b1c1b482bb3490e, 3bea48ecd7654efdabe998ee44e194b3
07:30:13.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:30:13.495035 9605 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
07:30:13.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:30:13.839329 9468 master_runner.cc:186] Error getting master registration for 127.8.210.190:36273: Network error: Client connection negotiation failed: client connection to 127.8.210.190:36273: connect: Connection refused (error 111),
07:30:13.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:30:13.839761 9468 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:30:13.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:30:13.861189 9555 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:30:13.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:30:13.861588 9555 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:30:14.265 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.210.189:45987(127.8.210.189:45987)] connecting to peer
07:30:14.265 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.210.190:36273(127.8.210.190:36273)] connecting to peer
07:30:14.266 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:30:14.266 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.210.189:45987(127.8.210.189:45987)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:30:14.266 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.210.189:45987: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:30:14.267 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:30:14.267 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.210.190:36273(127.8.210.190:36273)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:30:14.267 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.210.190:36273: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:30:14.269 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193; will retry
07:30:14.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:30:14.864471 9555 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:30:14.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:30:14.864849 9555 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:30:15.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:30:15.868010 9555 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:30:15.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:30:15.868258 9555 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:30:16.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:30:16.870993 9555 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:30:16.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:30:16.871235 9555 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:30:17.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:30:17.874128 9555 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:30:17.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:30:17.874365 9555 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:30:17.944 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.210.189:45987(127.8.210.189:45987)] connecting to peer
07:30:17.945 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.210.190:36273(127.8.210.190:36273)] connecting to peer
07:30:17.945 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:30:17.945 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.210.189:45987(127.8.210.189:45987)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:30:17.946 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.210.189:45987: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:30:17.946 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:30:17.947 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.210.190:36273(127.8.210.190:36273)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:30:17.947 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.210.190:36273: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:30:17.949 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193; will retry
07:30:18.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:30:18.842082 9468 master_runner.cc:186] Error getting master registration for 127.8.210.190:36273: Network error: Client connection negotiation failed: client connection to 127.8.210.190:36273: connect: Connection refused (error 111),
07:30:18.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:30:18.842442 9468 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:30:18.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:30:18.877163 9555 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:30:18.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:30:18.877408 9555 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:30:19.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:30:19.880563 9555 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:30:19.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:30:19.880786 9555 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:30:20.324 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.210.189:45987(127.8.210.189:45987)] connecting to peer
07:30:20.325 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.210.190:36273(127.8.210.190:36273)] connecting to peer
07:30:20.325 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:30:20.325 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.210.189:45987(127.8.210.189:45987)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:30:20.326 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.210.189:45987: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:30:20.328 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:30:20.328 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.210.190:36273(127.8.210.190:36273)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:30:20.328 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.210.190:36273: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:30:20.330 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193; will retry
07:30:20.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:30:20.883661 9555 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:30:20.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:30:20.883901 9555 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:30:21.284 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.210.189:45987(127.8.210.189:45987)] connecting to peer
07:30:21.285 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.210.190:36273(127.8.210.190:36273)] connecting to peer
07:30:21.286 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:30:21.286 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.210.189:45987(127.8.210.189:45987)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:30:21.286 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.210.189:45987: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:30:21.286 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:30:21.287 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.210.190:36273(127.8.210.190:36273)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:30:21.287 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.210.190:36273: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:30:21.289 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193; will retry
07:30:21.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:30:21.886715 9555 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:30:21.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:30:21.887023 9555 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:30:22.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:30:22.890381 9555 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 6107c62b44c140eba62f248d278c05d2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:30:22.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:30:22.890681 9555 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:30:23.164 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.210.189:45987(127.8.210.189:45987)] connecting to peer
07:30:23.165 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.210.190:36273(127.8.210.190:36273)] connecting to peer
07:30:23.165 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:30:23.166 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.210.189:45987(127.8.210.189:45987)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:30:23.166 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.210.189:45987: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987
07:30:23.166 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:30:23.167 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.210.190:36273(127.8.210.190:36273)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:30:23.167 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.210.190:36273: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
07:30:23.169 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193; will retry
07:30:23.174 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=29526), Traces: [0ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.189:45987, [135ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.190:36273, [137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.188:35193, [200ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.190:36273: Network error: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [209ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.189:45987: Network error: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987, [576ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.188:35193: OK, [582ms] delaying RPC due to: Service unavailable: Master config (127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987, [598ms] refreshing cache from master, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.189:45987, [602ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.190:36273, [604ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.188:35193, [604ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.189:45987: Network error: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987, [615ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.190:36273: Network error: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [618ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.188:35193: OK, [619ms] delaying RPC due to: Service unavailable: Master config (127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [637ms] refreshing cache from master, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.189:45987, [639ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.190:36273, [639ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.188:35193, [641ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.189:45987: Network error: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987, [643ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.190:36273: Network error: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [645ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.188:35193: OK, [646ms] delaying RPC due to: Service unavailable: Master config (127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [657ms] refreshing cache from master, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.189:45987, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.190:36273, [659ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.188:35193, [660ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.189:45987: Network error: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987, [664ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.190:36273: Network error: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [665ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.188:35193: OK, [666ms] delaying RPC due to: Service unavailable: Master config (127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [677ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.189:45987, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.190:36273, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.188:35193, [680ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.189:45987: Network error: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987, [684ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.190:36273: Network error: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [687ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.188:35193: OK, [688ms] delaying RPC due to: Service unavailable: Master config (127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [717ms] refreshing cache from master, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.189:45987, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.190:36273, [719ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.188:35193, [720ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.189:45987: Network error: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987, [727ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.188:35193: OK, [731ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.190:36273: Network error: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [732ms] delaying RPC due to: Service unavailable: Master config (127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [797ms] refreshing cache from master, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.189:45987, [799ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.190:36273, [801ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.188:35193, [801ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.189:45987: Network error: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987, [805ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.190:36273: Network error: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [807ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.188:35193: OK, [808ms] delaying RPC due to: Service unavailable: Master config (127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [837ms] refreshing cache from master, [838ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.189:45987, [840ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.190:36273, [841ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.188:35193, [841ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.189:45987: Network error: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987, [843ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.190:36273: Network error: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [846ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.188:35193: OK, [847ms] delaying RPC due to: Service unavailable: Master config (127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [997ms] refreshing cache from master, [998ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.189:45987, [999ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.190:36273, [1000ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.188:35193, [1001ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.189:45987: Network error: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987, [1005ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.190:36273: Network error: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [1007ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.188:35193: OK, [1007ms] delaying RPC due to: Service unavailable: Master config (127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [1277ms] refreshing cache from master, [1277ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.189:45987, [1279ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.190:36273, [1280ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.189:45987: Network error: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987, [1280ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.188:35193, [1281ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.190:36273: Network error: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [1285ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.188:35193: OK, [1286ms] delaying RPC due to: Service unavailable: Master config (127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [1817ms] refreshing cache from master, [1817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.189:45987, [1819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.190:36273, [1819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.188:35193, [1820ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.189:45987: Network error: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987, [1823ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.190:36273: Network error: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [1824ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.188:35193: OK, [1825ms] delaying RPC due to: Service unavailable: Master config (127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [2457ms] refreshing cache from master, [2458ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.189:45987, [2459ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.190:36273, [2460ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.188:35193, [2461ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.189:45987: Network error: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987, [2462ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.190:36273: Network error: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [2464ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.188:35193: OK, [2464ms] delaying RPC due to: Service unavailable: Master config (127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [4777ms] refreshing cache from master, [4778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.189:45987, [4779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.190:36273, [4779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.188:35193, [4780ms] too many traces: truncated at 100 traces, deferred=Deferred@1084468977(state=PENDING, result=null, callback=(continuation of Deferred@1023325882 after retry RPC after error@169674394) -> (continuation of Deferred@993257247 after retry RPC after error@1715077806) -> (continuation of Deferred@800596917 after retry RPC after error@1307822350) -> (continuation of Deferred@1776075952 after retry RPC after error@1598391383) -> (continuation of Deferred@831431041 after retry RPC after error@1650894087) -> (continuation of Deferred@1838159039 after retry RPC after error@511984230) -> (continuation of Deferred@1514302110 after retry RPC after error@224113228) -> (continuation of Deferred@1619225268 after retry RPC after error@1546163120) -> (continuation of Deferred@360121393 after retry RPC after error@1164840193) -> (continuation of Deferred@1044584914 after retry RPC after error@644786361) -> (continuation of Deferred@185956041 after retry RPC after error@1042707631) -> (continuation of Deferred@403535664 after retry RPC after error@1265971176) -> (continuation of Deferred@1802939249 after retry RPC after error@1727539644) -> (continuation of Deferred@364865188 after retry RPC after error@2010583444) -> (continuation of Deferred@690225468 after retry RPC after error@238450797) -> (continuation of Deferred@1624417609 after retry RPC after error@801526646) -> (continuation of Deferred@652380755 after retry RPC after error@455769074) -> (continuation of Deferred@1856470323 after retry RPC after error@1936186994) -> (continuation of Deferred@1097167455 after retry RPC after error@156791914) -> (continuation of Deferred@1556340375 after retry RPC after error@286924595) -> (continuation of Deferred@2064085189 after retry RPC after error@825663723) -> (continuation of Deferred@1942218556 after retry RPC after error@845254128) -> (continuation of Deferred@1923109409 after retry RPC after error@1989514598) -> (continuation of Deferred@436430479 after retry RPC after error@607348274) -> (continuation of Deferred@1416759759 after retry RPC after error@244750466) -> (continuation of Deferred@262185153 after retry RPC after error@807224817), errback=(continuation of Deferred@1023325882 after retry RPC after error@169674394) -> (continuation of Deferred@993257247 after retry RPC after error@1715077806) -> (continuation of Deferred@800596917 after retry RPC after error@1307822350) -> (continuation of Deferred@1776075952 after retry RPC after error@1598391383) -> (continuation of Deferred@831431041 after retry RPC after error@1650894087) -> (continuation of Deferred@1838159039 after retry RPC after error@511984230) -> (continuation of Deferred@1514302110 after retry RPC after error@224113228) -> (continuation of Deferred@1619225268 after retry RPC after error@1546163120) -> (continuation of Deferred@360121393 after retry RPC after error@1164840193) -> (continuation of Deferred@1044584914 after retry RPC after error@644786361) -> (continuation of Deferred@185956041 after retry RPC after error@1042707631) -> (continuation of Deferred@403535664 after retry RPC after error@1265971176) -> (continuation of Deferred@1802939249 after retry RPC after error@1727539644) -> (continuation of Deferred@364865188 after retry RPC after error@2010583444) -> (continuation of Deferred@690225468 after retry RPC after error@238450797) -> (continuation of Deferred@1624417609 after retry RPC after error@801526646) -> (continuation of Deferred@652380755 after retry RPC after error@455769074) -> (continuation of Deferred@1856470323 after retry RPC after error@1936186994) -> (continuation of Deferred@1097167455 after retry RPC after error@156791914) -> (continuation of Deferred@1556340375 after retry RPC after error@286924595) -> (continuation of Deferred@2064085189 after retry RPC after error@825663723) -> (continuation of Deferred@1942218556 after retry RPC after error@845254128) -> (continuation of Deferred@1923109409 after retry RPC after error@1989514598) -> (continuation of Deferred@436430479 after retry RPC after error@607348274) -> (continuation of Deferred@1416759759 after retry RPC after error@244750466) -> (continuation of Deferred@262185153 after retry RPC after error@807224817)))
07:30:23.178 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.8.210.190:36273(127.8.210.190:36273)] disconnecting while in state TERMINATED
07:30:23.179 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.8.210.189:45987(127.8.210.189:45987)] disconnecting while in state TERMINATED
07:30:23.180 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.8.210.188:35193(127.8.210.188:35193)] disconnecting while in state READY
07:30:23.184 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
07:30:23.186 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.8.210.188:35193(127.8.210.188:35193)] handling channelInactive
07:30:23.186 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.210.188:35193(127.8.210.188:35193)] cleaning up while in state READY due to: connection closed
07:30:23.188 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
07:30:23.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:30:23.195158 9034 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task8eqvEo/build/tsan/bin/kudu with pid 9468
07:30:24.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-24T07:30:24Z chronyd exiting
07:30:25.426 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
07:30:25.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:30:25.428200 8304 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task8eqvEo/build/tsan/bin/kudu with pid 8585
07:30:25.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:30:25.452805 8304 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task8eqvEo/build/tsan/bin/kudu with pid 8742
07:30:25.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:30:25.477681 8304 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task8eqvEo/build/tsan/bin/kudu with pid 8887
07:30:25.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:30:25.508809 8304 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task8eqvEo/build/tsan/bin/kudu with pid 8316
07:30:25.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:30:25.540205 8304 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task8eqvEo/build/tsan/bin/kudu with pid 8386
07:30:25.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:30:25.548913 8486 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.28.62:45427: connect: Connection refused (error 111)
07:30:25.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:30:25.552073 8486 consensus_peers.cc:487] T 00000000000000000000000000000000 P d9518af1d5b74701849ed730f86695b4 -> Peer c25a52d16dc24d558d31e61573570268 (127.8.28.62:45427): Couldn't send request to peer c25a52d16dc24d558d31e61573570268. Status: Network error: Client connection negotiation failed: client connection to 127.8.28.62:45427: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
07:30:25.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:30:25.571595 8488 connection.cc:535] client connection to 127.8.28.61:34477 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
07:30:25.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:30:25.572736 8304 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task8eqvEo/build/tsan/bin/kudu with pid 8461
07:30:26.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-24T07:30:26Z chronyd exiting
07:30:27.746 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=29526), Traces: [0ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.189:45987, [135ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.190:36273, [137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.188:35193, [200ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.190:36273: Network error: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [209ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.189:45987: Network error: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987, [576ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.188:35193: OK, [582ms] delaying RPC due to: Service unavailable: Master config (127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987, [598ms] refreshing cache from master, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.189:45987, [602ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.190:36273, [604ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.188:35193, [604ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.189:45987: Network error: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987, [615ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.190:36273: Network error: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [618ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.188:35193: OK, [619ms] delaying RPC due to: Service unavailable: Master config (127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [637ms] refreshing cache from master, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.189:45987, [639ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.190:36273, [639ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.188:35193, [641ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.189:45987: Network error: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987, [643ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.190:36273: Network error: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [645ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.188:35193: OK, [646ms] delaying RPC due to: Service unavailable: Master config (127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [657ms] refreshing cache from master, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.189:45987, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.190:36273, [659ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.188:35193, [660ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.189:45987: Network error: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987, [664ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.190:36273: Network error: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [665ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.188:35193: OK, [666ms] delaying RPC due to: Service unavailable: Master config (127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [677ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.189:45987, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.190:36273, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.188:35193, [680ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.189:45987: Network error: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987, [684ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.190:36273: Network error: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [687ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.188:35193: OK, [688ms] delaying RPC due to: Service unavailable: Master config (127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [717ms] refreshing cache from master, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.189:45987, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.190:36273, [719ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.188:35193, [720ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.189:45987: Network error: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987, [727ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.188:35193: OK, [731ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.190:36273: Network error: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [732ms] delaying RPC due to: Service unavailable: Master config (127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [797ms] refreshing cache from master, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.189:45987, [799ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.190:36273, [801ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.188:35193, [801ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.189:45987: Network error: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987, [805ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.190:36273: Network error: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [807ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.188:35193: OK, [808ms] delaying RPC due to: Service unavailable: Master config (127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [837ms] refreshing cache from master, [838ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.189:45987, [840ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.190:36273, [841ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.188:35193, [841ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.189:45987: Network error: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987, [843ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.190:36273: Network error: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [846ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.188:35193: OK, [847ms] delaying RPC due to: Service unavailable: Master config (127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [997ms] refreshing cache from master, [998ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.189:45987, [999ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.190:36273, [1000ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.188:35193, [1001ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.189:45987: Network error: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987, [1005ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.190:36273: Network error: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [1007ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.188:35193: OK, [1007ms] delaying RPC due to: Service unavailable: Master config (127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [1277ms] refreshing cache from master, [1277ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.189:45987, [1279ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.190:36273, [1280ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.189:45987: Network error: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987, [1280ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.188:35193, [1281ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.190:36273: Network error: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [1285ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.188:35193: OK, [1286ms] delaying RPC due to: Service unavailable: Master config (127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [1817ms] refreshing cache from master, [1817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.189:45987, [1819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.190:36273, [1819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.188:35193, [1820ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.189:45987: Network error: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987, [1823ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.190:36273: Network error: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [1824ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.188:35193: OK, [1825ms] delaying RPC due to: Service unavailable: Master config (127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [2457ms] refreshing cache from master, [2458ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.189:45987, [2459ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.190:36273, [2460ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.188:35193, [2461ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.189:45987: Network error: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987, [2462ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.190:36273: Network error: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [2464ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.188:35193: OK, [2464ms] delaying RPC due to: Service unavailable: Master config (127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [4777ms] refreshing cache from master, [4778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.189:45987, [4779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.190:36273, [4779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.188:35193, [4780ms] too many traces: truncated at 100 traces, deferred=Deferred@1084468977(state=PENDING, result=null, callback=(continuation of Deferred@1023325882 after retry RPC after error@169674394) -> (continuation of Deferred@993257247 after retry RPC after error@1715077806) -> (continuation of Deferred@800596917 after retry RPC after error@1307822350) -> (continuation of Deferred@1776075952 after retry RPC after error@1598391383) -> (continuation of Deferred@831431041 after retry RPC after error@1650894087) -> (continuation of Deferred@1838159039 after retry RPC after error@511984230) -> (continuation of Deferred@1514302110 after retry RPC after error@224113228) -> (continuation of Deferred@1619225268 after retry RPC after error@1546163120) -> (continuation of Deferred@360121393 after retry RPC after error@1164840193) -> (continuation of Deferred@1044584914 after retry RPC after error@644786361) -> (continuation of Deferred@185956041 after retry RPC after error@1042707631) -> (continuation of Deferred@403535664 after retry RPC after error@1265971176) -> (continuation of Deferred@1802939249 after retry RPC after error@1727539644) -> (continuation of Deferred@364865188 after retry RPC after error@2010583444) -> (continuation of Deferred@690225468 after retry RPC after error@238450797) -> (continuation of Deferred@1624417609 after retry RPC after error@801526646) -> (continuation of Deferred@652380755 after retry RPC after error@455769074) -> (continuation of Deferred@1856470323 after retry RPC after error@1936186994) -> (continuation of Deferred@1097167455 after retry RPC after error@156791914) -> (continuation of Deferred@1556340375 after retry RPC after error@286924595) -> (continuation of Deferred@2064085189 after retry RPC after error@825663723) -> (continuation of Deferred@1942218556 after retry RPC after error@845254128) -> (continuation of Deferred@1923109409 after retry RPC after error@1989514598) -> (continuation of Deferred@436430479 after retry RPC after error@607348274) -> (continuation of Deferred@1416759759 after retry RPC after error@244750466) -> (continuation of Deferred@262185153 after retry RPC after error@807224817), errback=(continuation of Deferred@1023325882 after retry RPC after error@169674394) -> (continuation of Deferred@993257247 after retry RPC after error@1715077806) -> (continuation of Deferred@800596917 after retry RPC after error@1307822350) -> (continuation of Deferred@1776075952 after retry RPC after error@1598391383) -> (continuation of Deferred@831431041 after retry RPC after error@1650894087) -> (continuation of Deferred@1838159039 after retry RPC after error@511984230) -> (continuation of Deferred@1514302110 after retry RPC after error@224113228) -> (continuation of Deferred@1619225268 after retry RPC after error@1546163120) -> (continuation of Deferred@360121393 after retry RPC after error@1164840193) -> (continuation of Deferred@1044584914 after retry RPC after error@644786361) -> (continuation of Deferred@185956041 after retry RPC after error@1042707631) -> (continuation of Deferred@403535664 after retry RPC after error@1265971176) -> (continuation of Deferred@1802939249 after retry RPC after error@1727539644) -> (continuation of Deferred@364865188 after retry RPC after error@2010583444) -> (continuation of Deferred@690225468 after retry RPC after error@238450797) -> (continuation of Deferred@1624417609 after retry RPC after error@801526646) -> (continuation of Deferred@652380755 after retry RPC after error@455769074) -> (continuation of Deferred@1856470323 after retry RPC after error@1936186994) -> (continuation of Deferred@1097167455 after retry RPC after error@156791914) -> (continuation of Deferred@1556340375 after retry RPC after error@286924595) -> (continuation of Deferred@2064085189 after retry RPC after error@825663723) -> (continuation of Deferred@1942218556 after retry RPC after error@845254128) -> (continuation of Deferred@1923109409 after retry RPC after error@1989514598) -> (continuation of Deferred@436430479 after retry RPC after error@607348274) -> (continuation of Deferred@1416759759 after retry RPC after error@244750466) -> (continuation of Deferred@262185153 after retry RPC after error@807224817)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.8.210.190:36273
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=29526), Traces: [0ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.189:45987, [135ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.190:36273, [137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.188:35193, [200ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.190:36273: Network error: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [209ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.189:45987: Network error: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987, [576ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.188:35193: OK, [582ms] delaying RPC due to: Service unavailable: Master config (127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987, [598ms] refreshing cache from master, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.189:45987, [602ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.190:36273, [604ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.188:35193, [604ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.189:45987: Network error: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987, [615ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.190:36273: Network error: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [618ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.188:35193: OK, [619ms] delaying RPC due to: Service unavailable: Master config (127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [637ms] refreshing cache from master, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.189:45987, [639ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.190:36273, [639ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.188:35193, [641ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.189:45987: Network error: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987, [643ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.190:36273: Network error: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [645ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.188:35193: OK, [646ms] delaying RPC due to: Service unavailable: Master config (127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [657ms] refreshing cache from master, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.189:45987, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.190:36273, [659ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.188:35193, [660ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.189:45987: Network error: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987, [664ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.190:36273: Network error: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [665ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.188:35193: OK, [666ms] delaying RPC due to: Service unavailable: Master config (127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [677ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.189:45987, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.190:36273, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.188:35193, [680ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.189:45987: Network error: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987, [684ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.190:36273: Network error: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [687ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.188:35193: OK, [688ms] delaying RPC due to: Service unavailable: Master config (127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [717ms] refreshing cache from master, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.189:45987, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.190:36273, [719ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.188:35193, [720ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.189:45987: Network error: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987, [727ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.188:35193: OK, [731ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.190:36273: Network error: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [732ms] delaying RPC due to: Service unavailable: Master config (127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [797ms] refreshing cache from master, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.189:45987, [799ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.190:36273, [801ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.188:35193, [801ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.189:45987: Network error: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987, [805ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.190:36273: Network error: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [807ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.188:35193: OK, [808ms] delaying RPC due to: Service unavailable: Master config (127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [837ms] refreshing cache from master, [838ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.189:45987, [840ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.190:36273, [841ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.188:35193, [841ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.189:45987: Network error: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987, [843ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.190:36273: Network error: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [846ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.188:35193: OK, [847ms] delaying RPC due to: Service unavailable: Master config (127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [997ms] refreshing cache from master, [998ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.189:45987, [999ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.190:36273, [1000ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.188:35193, [1001ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.189:45987: Network error: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987, [1005ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.190:36273: Network error: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [1007ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.188:35193: OK, [1007ms] delaying RPC due to: Service unavailable: Master config (127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [1277ms] refreshing cache from master, [1277ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.189:45987, [1279ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.190:36273, [1280ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.189:45987: Network error: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987, [1280ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.188:35193, [1281ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.190:36273: Network error: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [1285ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.188:35193: OK, [1286ms] delaying RPC due to: Service unavailable: Master config (127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [1817ms] refreshing cache from master, [1817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.189:45987, [1819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.190:36273, [1819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.188:35193, [1820ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.189:45987: Network error: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987, [1823ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.190:36273: Network error: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [1824ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.188:35193: OK, [1825ms] delaying RPC due to: Service unavailable: Master config (127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [2457ms] refreshing cache from master, [2458ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.189:45987, [2459ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.190:36273, [2460ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.188:35193, [2461ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.189:45987: Network error: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987, [2462ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.190:36273: Network error: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [2464ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.188:35193: OK, [2464ms] delaying RPC due to: Service unavailable: Master config (127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [4777ms] refreshing cache from master, [4778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.189:45987, [4779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.190:36273, [4779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.188:35193, [4780ms] too many traces: truncated at 100 traces, deferred=Deferred@1084468977(state=PENDING, result=null, callback=(continuation of Deferred@1023325882 after retry RPC after error@169674394) -> (continuation of Deferred@993257247 after retry RPC after error@1715077806) -> (continuation of Deferred@800596917 after retry RPC after error@1307822350) -> (continuation of Deferred@1776075952 after retry RPC after error@1598391383) -> (continuation of Deferred@831431041 after retry RPC after error@1650894087) -> (continuation of Deferred@1838159039 after retry RPC after error@511984230) -> (continuation of Deferred@1514302110 after retry RPC after error@224113228) -> (continuation of Deferred@1619225268 after retry RPC after error@1546163120) -> (continuation of Deferred@360121393 after retry RPC after error@1164840193) -> (continuation of Deferred@1044584914 after retry RPC after error@644786361) -> (continuation of Deferred@185956041 after retry RPC after error@1042707631) -> (continuation of Deferred@403535664 after retry RPC after error@1265971176) -> (continuation of Deferred@1802939249 after retry RPC after error@1727539644) -> (continuation of Deferred@364865188 after retry RPC after error@2010583444) -> (continuation of Deferred@690225468 after retry RPC after error@238450797) -> (continuation of Deferred@1624417609 after retry RPC after error@801526646) -> (continuation of Deferred@652380755 after retry RPC after error@455769074) -> (continuation of Deferred@1856470323 after retry RPC after error@1936186994) -> (continuation of Deferred@1097167455 after retry RPC after error@156791914) -> (continuation of Deferred@1556340375 after retry RPC after error@286924595) -> (continuation of Deferred@2064085189 after retry RPC after error@825663723) -> (continuation of Deferred@1942218556 after retry RPC after error@845254128) -> (continuation of Deferred@1923109409 after retry RPC after error@1989514598) -> (continuation of Deferred@436430479 after retry RPC after error@607348274) -> (continuation of Deferred@1416759759 after retry RPC after error@244750466) -> (continuation of Deferred@262185153 after retry RPC after error@807224817), errback=(continuation of Deferred@1023325882 after retry RPC after error@169674394) -> (continuation of Deferred@993257247 after retry RPC after error@1715077806) -> (continuation of Deferred@800596917 after retry RPC after error@1307822350) -> (continuation of Deferred@1776075952 after retry RPC after error@1598391383) -> (continuation of Deferred@831431041 after retry RPC after error@1650894087) -> (continuation of Deferred@1838159039 after retry RPC after error@511984230) -> (continuation of Deferred@1514302110 after retry RPC after error@224113228) -> (continuation of Deferred@1619225268 after retry RPC after error@1546163120) -> (continuation of Deferred@360121393 after retry RPC after error@1164840193) -> (continuation of Deferred@1044584914 after retry RPC after error@644786361) -> (continuation of Deferred@185956041 after retry RPC after error@1042707631) -> (continuation of Deferred@403535664 after retry RPC after error@1265971176) -> (continuation of Deferred@1802939249 after retry RPC after error@1727539644) -> (continuation of Deferred@364865188 after retry RPC after error@2010583444) -> (continuation of Deferred@690225468 after retry RPC after error@238450797) -> (continuation of Deferred@1624417609 after retry RPC after error@801526646) -> (continuation of Deferred@652380755 after retry RPC after error@455769074) -> (continuation of Deferred@1856470323 after retry RPC after error@1936186994) -> (continuation of Deferred@1097167455 after retry RPC after error@156791914) -> (continuation of Deferred@1556340375 after retry RPC after error@286924595) -> (continuation of Deferred@2064085189 after retry RPC after error@825663723) -> (continuation of Deferred@1942218556 after retry RPC after error@845254128) -> (continuation of Deferred@1923109409 after retry RPC after error@1989514598) -> (continuation of Deferred@436430479 after retry RPC after error@607348274) -> (continuation of Deferred@1416759759 after retry RPC after error@244750466) -> (continuation of Deferred@262185153 after retry RPC after error@807224817)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.8.210.190:36273
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
07:30:28.219 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
07:30:28.219 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=29526), Traces: [0ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.189:45987, [135ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.190:36273, [137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.188:35193, [200ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.190:36273: Network error: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [209ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.189:45987: Network error: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987, [576ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.188:35193: OK, [582ms] delaying RPC due to: Service unavailable: Master config (127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987, [598ms] refreshing cache from master, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.189:45987, [602ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.190:36273, [604ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.188:35193, [604ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.189:45987: Network error: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987, [615ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.190:36273: Network error: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [618ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.188:35193: OK, [619ms] delaying RPC due to: Service unavailable: Master config (127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [637ms] refreshing cache from master, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.189:45987, [639ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.190:36273, [639ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.188:35193, [641ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.189:45987: Network error: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987, [643ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.190:36273: Network error: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [645ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.188:35193: OK, [646ms] delaying RPC due to: Service unavailable: Master config (127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [657ms] refreshing cache from master, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.189:45987, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.190:36273, [659ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.188:35193, [660ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.189:45987: Network error: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987, [664ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.190:36273: Network error: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [665ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.188:35193: OK, [666ms] delaying RPC due to: Service unavailable: Master config (127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [677ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.189:45987, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.190:36273, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.188:35193, [680ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.189:45987: Network error: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987, [684ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.190:36273: Network error: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [687ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.188:35193: OK, [688ms] delaying RPC due to: Service unavailable: Master config (127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [717ms] refreshing cache from master, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.189:45987, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.190:36273, [719ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.188:35193, [720ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.189:45987: Network error: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987, [727ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.188:35193: OK, [731ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.190:36273: Network error: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [732ms] delaying RPC due to: Service unavailable: Master config (127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [797ms] refreshing cache from master, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.189:45987, [799ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.190:36273, [801ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.188:35193, [801ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.189:45987: Network error: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987, [805ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.190:36273: Network error: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [807ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.188:35193: OK, [808ms] delaying RPC due to: Service unavailable: Master config (127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [837ms] refreshing cache from master, [838ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.189:45987, [840ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.190:36273, [841ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.188:35193, [841ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.189:45987: Network error: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987, [843ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.190:36273: Network error: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [846ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.188:35193: OK, [847ms] delaying RPC due to: Service unavailable: Master config (127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [997ms] refreshing cache from master, [998ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.189:45987, [999ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.190:36273, [1000ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.188:35193, [1001ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.189:45987: Network error: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987, [1005ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.190:36273: Network error: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [1007ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.188:35193: OK, [1007ms] delaying RPC due to: Service unavailable: Master config (127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [1277ms] refreshing cache from master, [1277ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.189:45987, [1279ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.190:36273, [1280ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.189:45987: Network error: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987, [1280ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.188:35193, [1281ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.190:36273: Network error: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [1285ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.188:35193: OK, [1286ms] delaying RPC due to: Service unavailable: Master config (127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [1817ms] refreshing cache from master, [1817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.189:45987, [1819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.190:36273, [1819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.188:35193, [1820ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.189:45987: Network error: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987, [1823ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.190:36273: Network error: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [1824ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.188:35193: OK, [1825ms] delaying RPC due to: Service unavailable: Master config (127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [2457ms] refreshing cache from master, [2458ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.189:45987, [2459ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.190:36273, [2460ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.188:35193, [2461ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.189:45987: Network error: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987, [2462ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.190:36273: Network error: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [2464ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.188:35193: OK, [2464ms] delaying RPC due to: Service unavailable: Master config (127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [4777ms] refreshing cache from master, [4778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.189:45987, [4779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.190:36273, [4779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.188:35193, [4780ms] too many traces: truncated at 100 traces, deferred=Deferred@1084468977(state=PENDING, result=null, callback=(continuation of Deferred@1023325882 after retry RPC after error@169674394) -> (continuation of Deferred@993257247 after retry RPC after error@1715077806) -> (continuation of Deferred@800596917 after retry RPC after error@1307822350) -> (continuation of Deferred@1776075952 after retry RPC after error@1598391383) -> (continuation of Deferred@831431041 after retry RPC after error@1650894087) -> (continuation of Deferred@1838159039 after retry RPC after error@511984230) -> (continuation of Deferred@1514302110 after retry RPC after error@224113228) -> (continuation of Deferred@1619225268 after retry RPC after error@1546163120) -> (continuation of Deferred@360121393 after retry RPC after error@1164840193) -> (continuation of Deferred@1044584914 after retry RPC after error@644786361) -> (continuation of Deferred@185956041 after retry RPC after error@1042707631) -> (continuation of Deferred@403535664 after retry RPC after error@1265971176) -> (continuation of Deferred@1802939249 after retry RPC after error@1727539644) -> (continuation of Deferred@364865188 after retry RPC after error@2010583444) -> (continuation of Deferred@690225468 after retry RPC after error@238450797) -> (continuation of Deferred@1624417609 after retry RPC after error@801526646) -> (continuation of Deferred@652380755 after retry RPC after error@455769074) -> (continuation of Deferred@1856470323 after retry RPC after error@1936186994) -> (continuation of Deferred@1097167455 after retry RPC after error@156791914) -> (continuation of Deferred@1556340375 after retry RPC after error@286924595) -> (continuation of Deferred@2064085189 after retry RPC after error@825663723) -> (continuation of Deferred@1942218556 after retry RPC after error@845254128) -> (continuation of Deferred@1923109409 after retry RPC after error@1989514598) -> (continuation of Deferred@436430479 after retry RPC after error@607348274) -> (continuation of Deferred@1416759759 after retry RPC after error@244750466) -> (continuation of Deferred@262185153 after retry RPC after error@807224817), errback=(continuation of Deferred@1023325882 after retry RPC after error@169674394) -> (continuation of Deferred@993257247 after retry RPC after error@1715077806) -> (continuation of Deferred@800596917 after retry RPC after error@1307822350) -> (continuation of Deferred@1776075952 after retry RPC after error@1598391383) -> (continuation of Deferred@831431041 after retry RPC after error@1650894087) -> (continuation of Deferred@1838159039 after retry RPC after error@511984230) -> (continuation of Deferred@1514302110 after retry RPC after error@224113228) -> (continuation of Deferred@1619225268 after retry RPC after error@1546163120) -> (continuation of Deferred@360121393 after retry RPC after error@1164840193) -> (continuation of Deferred@1044584914 after retry RPC after error@644786361) -> (continuation of Deferred@185956041 after retry RPC after error@1042707631) -> (continuation of Deferred@403535664 after retry RPC after error@1265971176) -> (continuation of Deferred@1802939249 after retry RPC after error@1727539644) -> (continuation of Deferred@364865188 after retry RPC after error@2010583444) -> (continuation of Deferred@690225468 after retry RPC after error@238450797) -> (continuation of Deferred@1624417609 after retry RPC after error@801526646) -> (continuation of Deferred@652380755 after retry RPC after error@455769074) -> (continuation of Deferred@1856470323 after retry RPC after error@1936186994) -> (continuation of Deferred@1097167455 after retry RPC after error@156791914) -> (continuation of Deferred@1556340375 after retry RPC after error@286924595) -> (continuation of Deferred@2064085189 after retry RPC after error@825663723) -> (continuation of Deferred@1942218556 after retry RPC after error@845254128) -> (continuation of Deferred@1923109409 after retry RPC after error@1989514598) -> (continuation of Deferred@436430479 after retry RPC after error@607348274) -> (continuation of Deferred@1416759759 after retry RPC after error@244750466) -> (continuation of Deferred@262185153 after retry RPC after error@807224817)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.8.210.190:36273
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=29526), Traces: [0ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.189:45987, [135ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.190:36273, [137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.188:35193, [200ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.190:36273: Network error: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [209ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.189:45987: Network error: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987, [576ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.188:35193: OK, [582ms] delaying RPC due to: Service unavailable: Master config (127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987, [598ms] refreshing cache from master, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.189:45987, [602ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.190:36273, [604ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.188:35193, [604ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.189:45987: Network error: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987, [615ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.190:36273: Network error: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [618ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.188:35193: OK, [619ms] delaying RPC due to: Service unavailable: Master config (127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [637ms] refreshing cache from master, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.189:45987, [639ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.190:36273, [639ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.188:35193, [641ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.189:45987: Network error: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987, [643ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.190:36273: Network error: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [645ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.188:35193: OK, [646ms] delaying RPC due to: Service unavailable: Master config (127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [657ms] refreshing cache from master, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.189:45987, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.190:36273, [659ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.188:35193, [660ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.189:45987: Network error: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987, [664ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.190:36273: Network error: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [665ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.188:35193: OK, [666ms] delaying RPC due to: Service unavailable: Master config (127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [677ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.189:45987, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.190:36273, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.188:35193, [680ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.189:45987: Network error: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987, [684ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.190:36273: Network error: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [687ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.188:35193: OK, [688ms] delaying RPC due to: Service unavailable: Master config (127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [717ms] refreshing cache from master, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.189:45987, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.190:36273, [719ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.188:35193, [720ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.189:45987: Network error: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987, [727ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.188:35193: OK, [731ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.190:36273: Network error: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [732ms] delaying RPC due to: Service unavailable: Master config (127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [797ms] refreshing cache from master, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.189:45987, [799ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.190:36273, [801ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.188:35193, [801ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.189:45987: Network error: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987, [805ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.190:36273: Network error: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [807ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.188:35193: OK, [808ms] delaying RPC due to: Service unavailable: Master config (127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [837ms] refreshing cache from master, [838ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.189:45987, [840ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.190:36273, [841ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.188:35193, [841ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.189:45987: Network error: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987, [843ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.190:36273: Network error: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [846ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.188:35193: OK, [847ms] delaying RPC due to: Service unavailable: Master config (127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [997ms] refreshing cache from master, [998ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.189:45987, [999ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.190:36273, [1000ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.188:35193, [1001ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.189:45987: Network error: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987, [1005ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.190:36273: Network error: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [1007ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.188:35193: OK, [1007ms] delaying RPC due to: Service unavailable: Master config (127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [1277ms] refreshing cache from master, [1277ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.189:45987, [1279ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.190:36273, [1280ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.189:45987: Network error: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987, [1280ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.188:35193, [1281ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.190:36273: Network error: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [1285ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.188:35193: OK, [1286ms] delaying RPC due to: Service unavailable: Master config (127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [1817ms] refreshing cache from master, [1817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.189:45987, [1819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.190:36273, [1819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.188:35193, [1820ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.189:45987: Network error: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987, [1823ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.190:36273: Network error: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [1824ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.188:35193: OK, [1825ms] delaying RPC due to: Service unavailable: Master config (127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [2457ms] refreshing cache from master, [2458ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.189:45987, [2459ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.190:36273, [2460ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.188:35193, [2461ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.189:45987: Network error: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987, [2462ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.190:36273: Network error: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [2464ms] Sub RPC ConnectToMaster: received response from server master-127.8.210.188:35193: OK, [2464ms] delaying RPC due to: Service unavailable: Master config (127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273, [4777ms] refreshing cache from master, [4778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.189:45987, [4779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.190:36273, [4779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.210.188:35193, [4780ms] too many traces: truncated at 100 traces, deferred=Deferred@1084468977(state=PENDING, result=null, callback=(continuation of Deferred@1023325882 after retry RPC after error@169674394) -> (continuation of Deferred@993257247 after retry RPC after error@1715077806) -> (continuation of Deferred@800596917 after retry RPC after error@1307822350) -> (continuation of Deferred@1776075952 after retry RPC after error@1598391383) -> (continuation of Deferred@831431041 after retry RPC after error@1650894087) -> (continuation of Deferred@1838159039 after retry RPC after error@511984230) -> (continuation of Deferred@1514302110 after retry RPC after error@224113228) -> (continuation of Deferred@1619225268 after retry RPC after error@1546163120) -> (continuation of Deferred@360121393 after retry RPC after error@1164840193) -> (continuation of Deferred@1044584914 after retry RPC after error@644786361) -> (continuation of Deferred@185956041 after retry RPC after error@1042707631) -> (continuation of Deferred@403535664 after retry RPC after error@1265971176) -> (continuation of Deferred@1802939249 after retry RPC after error@1727539644) -> (continuation of Deferred@364865188 after retry RPC after error@2010583444) -> (continuation of Deferred@690225468 after retry RPC after error@238450797) -> (continuation of Deferred@1624417609 after retry RPC after error@801526646) -> (continuation of Deferred@652380755 after retry RPC after error@455769074) -> (continuation of Deferred@1856470323 after retry RPC after error@1936186994) -> (continuation of Deferred@1097167455 after retry RPC after error@156791914) -> (continuation of Deferred@1556340375 after retry RPC after error@286924595) -> (continuation of Deferred@2064085189 after retry RPC after error@825663723) -> (continuation of Deferred@1942218556 after retry RPC after error@845254128) -> (continuation of Deferred@1923109409 after retry RPC after error@1989514598) -> (continuation of Deferred@436430479 after retry RPC after error@607348274) -> (continuation of Deferred@1416759759 after retry RPC after error@244750466) -> (continuation of Deferred@262185153 after retry RPC after error@807224817), errback=(continuation of Deferred@1023325882 after retry RPC after error@169674394) -> (continuation of Deferred@993257247 after retry RPC after error@1715077806) -> (continuation of Deferred@800596917 after retry RPC after error@1307822350) -> (continuation of Deferred@1776075952 after retry RPC after error@1598391383) -> (continuation of Deferred@831431041 after retry RPC after error@1650894087) -> (continuation of Deferred@1838159039 after retry RPC after error@511984230) -> (continuation of Deferred@1514302110 after retry RPC after error@224113228) -> (continuation of Deferred@1619225268 after retry RPC after error@1546163120) -> (continuation of Deferred@360121393 after retry RPC after error@1164840193) -> (continuation of Deferred@1044584914 after retry RPC after error@644786361) -> (continuation of Deferred@185956041 after retry RPC after error@1042707631) -> (continuation of Deferred@403535664 after retry RPC after error@1265971176) -> (continuation of Deferred@1802939249 after retry RPC after error@1727539644) -> (continuation of Deferred@364865188 after retry RPC after error@2010583444) -> (continuation of Deferred@690225468 after retry RPC after error@238450797) -> (continuation of Deferred@1624417609 after retry RPC after error@801526646) -> (continuation of Deferred@652380755 after retry RPC after error@455769074) -> (continuation of Deferred@1856470323 after retry RPC after error@1936186994) -> (continuation of Deferred@1097167455 after retry RPC after error@156791914) -> (continuation of Deferred@1556340375 after retry RPC after error@286924595) -> (continuation of Deferred@2064085189 after retry RPC after error@825663723) -> (continuation of Deferred@1942218556 after retry RPC after error@845254128) -> (continuation of Deferred@1923109409 after retry RPC after error@1989514598) -> (continuation of Deferred@436430479 after retry RPC after error@607348274) -> (continuation of Deferred@1416759759 after retry RPC after error@244750466) -> (continuation of Deferred@262185153 after retry RPC after error@807224817)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.8.210.189:45987,127.8.210.190:36273,127.8.210.188:35193) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.189:45987(127.8.210.189:45987): Connection refused: /127.8.210.189:45987,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.210.190:36273(127.8.210.190:36273): Connection refused: /127.8.210.190:36273
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.8.210.190:36273
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)