00:10:47.642 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
00:10:47.654 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task3KSwpE/test-tmp
00:10:47.657 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
00:10:47.661 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task3KSwpE/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
00:10:47.906 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
00:10:48.047 [DEBUG - main] (MiniKuduCluster.java:195) Response:
00:10:48.048 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
00:10:48.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:48.053690 7499 mini_oidc.cc:152] Starting JWKS server
00:10:48.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:48.058313 7499 webserver.cc:460] Webserver started at http://127.0.0.1:42803/ using document root <none> and password file <none>
00:10:48.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:48.058993 7499 mini_oidc.cc:189] Starting OIDC Discovery server
00:10:48.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:48.061280 7499 webserver.cc:460] Webserver started at http://127.0.0.1:41623/ using document root <none> and password file <none>
00:10:48.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-10T00:10:48Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
00:10:48.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-10T00:10:48Z Disabled control of system clock
00:10:48.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:48.125695 7499 external_mini_cluster.cc:1326] Running /tmp/dist-test-task3KSwpE/build/tsan/bin/kudu
00:10:48.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task3KSwpE/build/tsan/bin/kudu
00:10:48.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/master-0/wal
00:10:48.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/master-0/data
00:10:48.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:10:48.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:10:48.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:10:48.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:10:48.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:10:48.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:10:48.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/master-0/logs
00:10:48.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/master-0/data/info.pb
00:10:48.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:10:48.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:10:48.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:10:48.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:10:48.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:10:48.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:10:48.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:10:48.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:10:48.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
00:10:48.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:10:48.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:10:48.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:10:48.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.82.254:41719
00:10:48.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.82.254
00:10:48.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:10:48.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.82.212:46327
00:10:48.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:10:48.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:10:48.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:10:48.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:10:48.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.82.254:41719,127.7.82.253:34963,127.7.82.252:36925
00:10:48.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:10:48.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42803/jwks.json with env {}
00:10:48.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:48.435702 7515 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
00:10:48.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:48.436396 7515 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:10:48.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:48.437156 7515 flags.cc:425] Enabled unsafe flag: --never_fsync=true
00:10:48.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:48.468483 7515 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
00:10:48.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:48.468868 7515 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
00:10:48.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:48.469127 7515 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
00:10:48.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:48.469377 7515 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
00:10:48.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:48.469667 7515 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
00:10:48.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:48.469890 7515 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42803/jwks.json
00:10:48.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:48.506389 7515 master_runner.cc:386] Master server non-default flags:
00:10:48.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:10:48.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.82.212:46327
00:10:48.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:10:48.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:10:48.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/master-0/data
00:10:48.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/master-0/wal
00:10:48.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:10:48.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.82.254:41719,127.7.82.253:34963,127.7.82.252:36925
00:10:48.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:10:48.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:10:48.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:10:48.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.82.254:41719
00:10:48.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:10:48.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:10:48.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:10:48.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42803/jwks.json
00:10:48.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:10:48.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:10:48.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/master-0/data/info.pb
00:10:48.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.82.254
00:10:48.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:10:48.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:10:48.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:10:48.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:10:48.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:10:48.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:10:48.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/master-0/logs
00:10:48.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:10:48.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:10:48.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
00:10:48.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
00:10:48.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
00:10:48.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
00:10:48.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:10:48.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 23:15:58 UTC on ccycloud.quasar-ddmxrv.root.comops.site
00:10:48.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4576820
00:10:48.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
00:10:48.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:48.507908 7515 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:10:48.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:48.510224 7515 file_cache.cc:493] Constructed file cache file cache with capacity 419430
00:10:48.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:48.529610 7521 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:10:49.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:48.529845 7524 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:10:49.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:48.530458 7522 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:10:49.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:48.531579 7515 server_base.cc:1034] running on GCE node
00:10:49.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:49.790834 7515 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:10:49.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:49.793702 7515 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:10:49.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:49.795065 7515 hybrid_clock.cc:648] HybridClock initialized: now 1786320649795007 us; error 107 us; skew 500 ppm
00:10:49.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:49.795876 7515 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:10:49.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:49.806519 7515 webserver.cc:460] Webserver started at http://127.7.82.254:45777/ using document root <none> and password file <none>
00:10:49.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:49.807672 7515 fs_manager.cc:359] Metadata directory not provided
00:10:49.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:49.807865 7515 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:10:49.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:49.808342 7515 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
00:10:49.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:49.813400 7515 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/master-0/data/instance:
00:10:49.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8e86a0326ccf4373bd74a91e091575ae"
00:10:49.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-10 00:10:49 on dist-test-slave-cbbz"
00:10:49.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:49.814594 7515 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/master-0/wal/instance:
00:10:49.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8e86a0326ccf4373bd74a91e091575ae"
00:10:49.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-10 00:10:49 on dist-test-slave-cbbz"
00:10:49.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:49.822083 7515 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.003s sys 0.006s
00:10:49.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:49.827749 7532 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
00:10:49.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:49.828876 7515 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
00:10:49.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:49.829174 7515 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/master-0/data,/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/master-0/wal
00:10:49.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8e86a0326ccf4373bd74a91e091575ae"
00:10:49.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-10 00:10:49 on dist-test-slave-cbbz"
00:10:49.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:49.829607 7515 fs_report.cc:383] FS layout report
00:10:49.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:10:49.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/master-0/wal
00:10:49.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/master-0/wal
00:10:49.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/master-0/data/data
00:10:49.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:10:49.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:10:49.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:10:49.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:10:49.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:10:49.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:10:49.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:10:49.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:10:49.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:10:49.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:10:49.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:10:49.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:49.871538 7515 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
00:10:49.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:49.873169 7515 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:10:49.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:49.873585 7515 kserver.cc:163] Server-wide thread pool size limit: 3276
00:10:49.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:49.946746 7515 rpc_server.cc:307] RPC server started. Bound to: 127.7.82.254:41719
00:10:49.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:49.946884 7585 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.82.254:41719 every 8 connection(s)
00:10:49.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:49.949829 7515 server_base.cc:1166] Dumped server information to /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/master-0/data/info.pb
00:10:49.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:49.950850 7499 external_mini_cluster.cc:1388] Started /tmp/dist-test-task3KSwpE/build/tsan/bin/kudu as pid 7515
00:10:49.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:49.951406 7499 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/master-0/wal/instance
00:10:49.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:49.956640 7586 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
00:10:49.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:49.956905 7499 external_mini_cluster.cc:1326] Running /tmp/dist-test-task3KSwpE/build/tsan/bin/kudu
00:10:49.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task3KSwpE/build/tsan/bin/kudu
00:10:49.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/master-1/wal
00:10:49.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/master-1/data
00:10:49.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:10:49.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:10:49.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:10:49.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:10:49.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:10:49.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:10:49.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/master-1/logs
00:10:49.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/master-1/data/info.pb
00:10:49.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:10:49.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:10:49.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:10:49.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:10:49.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:10:49.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:10:49.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:10:49.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:10:49.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
00:10:49.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:10:49.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:10:49.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:10:49.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.82.253:34963
00:10:49.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.82.253
00:10:49.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:10:49.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.82.212:46327
00:10:49.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:10:49.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:10:49.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:10:49.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:10:49.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.82.254:41719,127.7.82.253:34963,127.7.82.252:36925
00:10:49.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:10:49.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42803/jwks.json with env {}
00:10:49.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:49.971295 7586 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.82.254" port: 41719 } has no permanent_uuid. Determining permanent_uuid...
00:10:50.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:49.999717 7586 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.82.253" port: 34963 } has no permanent_uuid. Determining permanent_uuid...
00:10:50.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:50.002188 7533 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.82.253:34963: connect: Connection refused (error 111)
00:10:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:50.006517 7586 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.82.253:34963: Network error: Client connection negotiation failed: client connection to 127.7.82.253:34963: connect: Connection refused (error 111)
00:10:50.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:50.056100 7586 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.82.253" port: 34963 } attempt: 1
00:10:50.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:50.060316 7586 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.82.253:34963: Network error: Client connection negotiation failed: client connection to 127.7.82.253:34963: connect: Connection refused (error 111)
00:10:50.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:50.128712 7586 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.82.253" port: 34963 } attempt: 2
00:10:50.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:50.132293 7586 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.82.253:34963: Network error: Client connection negotiation failed: client connection to 127.7.82.253:34963: connect: Connection refused (error 111)
00:10:50.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:50.223759 7586 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.82.253" port: 34963 } attempt: 3
00:10:50.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:50.227867 7586 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.82.253:34963: Network error: Client connection negotiation failed: client connection to 127.7.82.253:34963: connect: Connection refused (error 111)
00:10:50.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:50.271904 7588 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
00:10:50.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:50.272475 7588 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:10:50.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:50.273022 7588 flags.cc:425] Enabled unsafe flag: --never_fsync=true
00:10:50.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:50.303680 7588 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
00:10:50.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:50.304064 7588 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
00:10:50.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:50.304406 7588 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
00:10:50.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:50.304751 7588 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
00:10:50.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:50.305112 7588 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
00:10:50.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:50.305392 7588 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42803/jwks.json
00:10:50.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:50.341760 7588 master_runner.cc:386] Master server non-default flags:
00:10:50.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:10:50.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.82.212:46327
00:10:50.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:10:50.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:10:50.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/master-1/data
00:10:50.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/master-1/wal
00:10:50.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:10:50.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.82.254:41719,127.7.82.253:34963,127.7.82.252:36925
00:10:50.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:10:50.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:10:50.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:10:50.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.82.253:34963
00:10:50.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:10:50.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:10:50.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:10:50.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42803/jwks.json
00:10:50.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:10:50.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:10:50.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/master-1/data/info.pb
00:10:50.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.82.253
00:10:50.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:10:50.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:10:50.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:10:50.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:10:50.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:10:50.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:10:50.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/master-1/logs
00:10:50.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:10:50.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:10:50.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
00:10:50.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
00:10:50.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
00:10:50.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
00:10:50.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:10:50.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 23:15:58 UTC on ccycloud.quasar-ddmxrv.root.comops.site
00:10:50.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4576820
00:10:50.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
00:10:50.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:50.343333 7588 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:10:50.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:50.345685 7588 file_cache.cc:493] Constructed file cache file cache with capacity 419430
00:10:50.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:50.364895 7598 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:10:50.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:50.371277 7586 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.82.253" port: 34963 } attempt: 4
00:10:50.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:50.375325 7586 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.82.253:34963: Network error: Client connection negotiation failed: client connection to 127.7.82.253:34963: connect: Connection refused (error 111)
00:10:50.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:50.674713 7586 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.82.253" port: 34963 } attempt: 5
00:10:50.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:50.678979 7586 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.82.253:34963: Network error: Client connection negotiation failed: client connection to 127.7.82.253:34963: connect: Connection refused (error 111)
00:10:51.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:51.226387 7586 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.82.253" port: 34963 } attempt: 6
00:10:51.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:51.234230 7586 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.82.253:34963: Network error: Client connection negotiation failed: client connection to 127.7.82.253:34963: connect: Connection refused (error 111)
00:10:51.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:50.365768 7600 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:10:51.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:50.366029 7597 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:10:51.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:50.367349 7588 server_base.cc:1034] running on GCE node
00:10:51.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:51.569087 7588 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:10:51.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:51.572599 7588 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:10:51.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:51.574436 7588 hybrid_clock.cc:648] HybridClock initialized: now 1786320651574398 us; error 51 us; skew 500 ppm
00:10:51.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:51.575593 7588 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:10:51.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:51.583591 7588 webserver.cc:460] Webserver started at http://127.7.82.253:37189/ using document root <none> and password file <none>
00:10:51.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:51.584800 7588 fs_manager.cc:359] Metadata directory not provided
00:10:51.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:51.585099 7588 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:10:51.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:51.585744 7588 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
00:10:51.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:51.591897 7588 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/master-1/data/instance:
00:10:51.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1f3c256f83ed423daef2312b243ba2d9"
00:10:51.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-10 00:10:51 on dist-test-slave-cbbz"
00:10:51.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:51.593457 7588 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/master-1/wal/instance:
00:10:51.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1f3c256f83ed423daef2312b243ba2d9"
00:10:51.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-10 00:10:51 on dist-test-slave-cbbz"
00:10:51.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:51.601833 7588 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.003s sys 0.004s
00:10:51.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:51.607579 7610 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
00:10:51.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:51.608762 7588 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
00:10:51.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:51.609119 7588 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/master-1/data,/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/master-1/wal
00:10:51.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1f3c256f83ed423daef2312b243ba2d9"
00:10:51.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-10 00:10:51 on dist-test-slave-cbbz"
00:10:51.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:51.609519 7588 fs_report.cc:383] FS layout report
00:10:51.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:10:51.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/master-1/wal
00:10:51.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/master-1/wal
00:10:51.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/master-1/data/data
00:10:51.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:10:51.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:10:51.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:10:51.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:10:51.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:10:51.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:10:51.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:10:51.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:10:51.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:10:51.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:10:51.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:10:51.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:51.647255 7588 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
00:10:51.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:51.648896 7588 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:10:51.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:51.649452 7588 kserver.cc:163] Server-wide thread pool size limit: 3276
00:10:51.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:51.723814 7588 rpc_server.cc:307] RPC server started. Bound to: 127.7.82.253:34963
00:10:51.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:51.723922 7666 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.82.253:34963 every 8 connection(s)
00:10:51.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:51.726850 7588 server_base.cc:1166] Dumped server information to /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/master-1/data/info.pb
00:10:51.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:51.732128 7667 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
00:10:51.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:51.733287 7499 external_mini_cluster.cc:1388] Started /tmp/dist-test-task3KSwpE/build/tsan/bin/kudu as pid 7588
00:10:51.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:51.733636 7499 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/master-1/wal/instance
00:10:51.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:51.736030 7499 external_mini_cluster.cc:1326] Running /tmp/dist-test-task3KSwpE/build/tsan/bin/kudu
00:10:51.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task3KSwpE/build/tsan/bin/kudu
00:10:51.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/master-2/wal
00:10:51.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/master-2/data
00:10:51.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:10:51.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:10:51.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:10:51.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:10:51.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:10:51.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:10:51.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/master-2/logs
00:10:51.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/master-2/data/info.pb
00:10:51.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:10:51.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:10:51.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:10:51.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:10:51.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:10:51.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:10:51.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:10:51.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:10:51.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
00:10:51.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:10:51.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:10:51.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:10:51.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.82.252:36925
00:10:51.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.82.252
00:10:51.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:10:51.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.82.212:46327
00:10:51.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:10:51.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:10:51.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:10:51.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:10:51.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.82.254:41719,127.7.82.253:34963,127.7.82.252:36925
00:10:51.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:10:51.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42803/jwks.json with env {}
00:10:51.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:51.744562 7667 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.82.254" port: 41719 } has no permanent_uuid. Determining permanent_uuid...
00:10:51.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:51.767854 7667 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.82.253" port: 34963 } has no permanent_uuid. Determining permanent_uuid...
00:10:51.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:51.785665 7667 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.82.252" port: 36925 } has no permanent_uuid. Determining permanent_uuid...
00:10:51.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:51.787976 7611 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.82.252:36925: connect: Connection refused (error 111)
00:10:51.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:51.793361 7667 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.82.252:36925: Network error: Client connection negotiation failed: client connection to 127.7.82.252:36925: connect: Connection refused (error 111)
00:10:51.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:51.842931 7667 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.82.252" port: 36925 } attempt: 1
00:10:51.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:51.846943 7667 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.82.252:36925: Network error: Client connection negotiation failed: client connection to 127.7.82.252:36925: connect: Connection refused (error 111)
00:10:51.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:51.915320 7667 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.82.252" port: 36925 } attempt: 2
00:10:51.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:51.919109 7667 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.82.252:36925: Network error: Client connection negotiation failed: client connection to 127.7.82.252:36925: connect: Connection refused (error 111)
00:10:52.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:52.010524 7667 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.82.252" port: 36925 } attempt: 3
00:10:52.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:52.014312 7667 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.82.252:36925: Network error: Client connection negotiation failed: client connection to 127.7.82.252:36925: connect: Connection refused (error 111)
00:10:52.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:52.067212 7669 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
00:10:52.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:52.067906 7669 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:10:52.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:52.068626 7669 flags.cc:425] Enabled unsafe flag: --never_fsync=true
00:10:52.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:52.098932 7669 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
00:10:52.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:52.099300 7669 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
00:10:52.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:52.099642 7669 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
00:10:52.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:52.099974 7669 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
00:10:52.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:52.100307 7669 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
00:10:52.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:52.100600 7669 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42803/jwks.json
00:10:52.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:52.135635 7669 master_runner.cc:386] Master server non-default flags:
00:10:52.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:10:52.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.82.212:46327
00:10:52.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:10:52.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:10:52.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/master-2/data
00:10:52.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/master-2/wal
00:10:52.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:10:52.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.82.254:41719,127.7.82.253:34963,127.7.82.252:36925
00:10:52.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:10:52.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:10:52.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:10:52.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.82.252:36925
00:10:52.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:10:52.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:10:52.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:10:52.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42803/jwks.json
00:10:52.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:10:52.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:10:52.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/master-2/data/info.pb
00:10:52.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.82.252
00:10:52.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:10:52.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:10:52.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:10:52.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:10:52.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:10:52.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:10:52.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/master-2/logs
00:10:52.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:10:52.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:10:52.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
00:10:52.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
00:10:52.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
00:10:52.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
00:10:52.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:10:52.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 23:15:58 UTC on ccycloud.quasar-ddmxrv.root.comops.site
00:10:52.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4576820
00:10:52.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
00:10:52.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:52.137264 7669 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:10:52.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:52.139474 7669 file_cache.cc:493] Constructed file cache file cache with capacity 419430
00:10:52.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:52.157723 7667 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.82.252" port: 36925 } attempt: 4
00:10:52.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:52.158869 7680 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:10:52.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:52.161747 7667 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.82.252:36925: Network error: Client connection negotiation failed: client connection to 127.7.82.252:36925: connect: Connection refused (error 111)
00:10:52.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:52.294657 7586 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.82.253" port: 34963 } attempt: 7
00:10:52.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:52.307706 7586 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.82.252" port: 36925 } has no permanent_uuid. Determining permanent_uuid...
00:10:52.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:52.314427 7586 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.82.252:36925: Network error: Client connection negotiation failed: client connection to 127.7.82.252:36925: connect: Connection refused (error 111)
00:10:52.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:52.372843 7586 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.82.252" port: 36925 } attempt: 1
00:10:52.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:52.378059 7586 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.82.252:36925: Network error: Client connection negotiation failed: client connection to 127.7.82.252:36925: connect: Connection refused (error 111)
00:10:52.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:52.459462 7586 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.82.252" port: 36925 } attempt: 2
00:10:52.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:52.461115 7667 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.82.252" port: 36925 } attempt: 5
00:10:52.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:52.464116 7586 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.82.252:36925: Network error: Client connection negotiation failed: client connection to 127.7.82.252:36925: connect: Connection refused (error 111)
00:10:52.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:52.465317 7667 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.82.252:36925: Network error: Client connection negotiation failed: client connection to 127.7.82.252:36925: connect: Connection refused (error 111)
00:10:52.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:52.549609 7586 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.82.252" port: 36925 } attempt: 3
00:10:52.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:52.553952 7586 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.82.252:36925: Network error: Client connection negotiation failed: client connection to 127.7.82.252:36925: connect: Connection refused (error 111)
00:10:52.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:52.694597 7586 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.82.252" port: 36925 } attempt: 4
00:10:52.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:52.699312 7586 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.82.252:36925: Network error: Client connection negotiation failed: client connection to 127.7.82.252:36925: connect: Connection refused (error 111)
00:10:52.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:52.982934 7586 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.82.252" port: 36925 } attempt: 5
00:10:52.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:52.987789 7586 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.82.252:36925: Network error: Client connection negotiation failed: client connection to 127.7.82.252:36925: connect: Connection refused (error 111)
00:10:53.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.012722 7667 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.82.252" port: 36925 } attempt: 6
00:10:53.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:53.020577 7667 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.82.252:36925: Network error: Client connection negotiation failed: client connection to 127.7.82.252:36925: connect: Connection refused (error 111)
00:10:53.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:52.159691 7681 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:10:53.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:52.159873 7683 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:10:53.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:52.160728 7669 server_base.cc:1034] running on GCE node
00:10:53.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.377239 7669 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:10:53.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.380484 7669 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:10:53.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.381902 7669 hybrid_clock.cc:648] HybridClock initialized: now 1786320653381856 us; error 53 us; skew 500 ppm
00:10:53.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.382736 7669 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:10:53.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.389628 7669 webserver.cc:460] Webserver started at http://127.7.82.252:38247/ using document root <none> and password file <none>
00:10:53.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.390494 7669 fs_manager.cc:359] Metadata directory not provided
00:10:53.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.390673 7669 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:10:53.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.391043 7669 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
00:10:53.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.395838 7669 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/master-2/data/instance:
00:10:53.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c48b018d8a154bd8bd10fbf4739bcbc5"
00:10:53.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-10 00:10:53 on dist-test-slave-cbbz"
00:10:53.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.397091 7669 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/master-2/wal/instance:
00:10:53.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c48b018d8a154bd8bd10fbf4739bcbc5"
00:10:53.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-10 00:10:53 on dist-test-slave-cbbz"
00:10:53.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.404976 7669 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
00:10:53.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.411162 7697 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
00:10:53.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.412369 7669 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.005s
00:10:53.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.412647 7669 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/master-2/data,/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/master-2/wal
00:10:53.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c48b018d8a154bd8bd10fbf4739bcbc5"
00:10:53.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-10 00:10:53 on dist-test-slave-cbbz"
00:10:53.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.413074 7669 fs_report.cc:383] FS layout report
00:10:53.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:10:53.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/master-2/wal
00:10:53.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/master-2/wal
00:10:53.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/master-2/data/data
00:10:53.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:10:53.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:10:53.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:10:53.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:10:53.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:10:53.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:10:53.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:10:53.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:10:53.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:10:53.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:10:53.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:10:53.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.453929 7669 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
00:10:53.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.455533 7669 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:10:53.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.456085 7669 kserver.cc:163] Server-wide thread pool size limit: 3276
00:10:53.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.528903 7669 rpc_server.cc:307] RPC server started. Bound to: 127.7.82.252:36925
00:10:53.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.529013 7754 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.82.252:36925 every 8 connection(s)
00:10:53.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.531845 7669 server_base.cc:1166] Dumped server information to /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/master-2/data/info.pb
00:10:53.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.536659 7499 external_mini_cluster.cc:1388] Started /tmp/dist-test-task3KSwpE/build/tsan/bin/kudu as pid 7669
00:10:53.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.537102 7499 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/master-2/wal/instance
00:10:53.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.537726 7755 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
00:10:53.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.547386 7586 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.82.252" port: 36925 } attempt: 6
00:10:53.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.555035 7755 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.82.254" port: 41719 } has no permanent_uuid. Determining permanent_uuid...
00:10:53.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.586864 7755 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.82.253" port: 34963 } has no permanent_uuid. Determining permanent_uuid...
00:10:53.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.596529 7586 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8e86a0326ccf4373bd74a91e091575ae: Bootstrap starting.
00:10:53.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.603933 7586 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 8e86a0326ccf4373bd74a91e091575ae: Neither blocks nor log segments found. Creating new log.
00:10:53.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.606225 7755 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.82.252" port: 36925 } has no permanent_uuid. Determining permanent_uuid...
00:10:53.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.606271 7586 log.cc:824] T 00000000000000000000000000000000 P 8e86a0326ccf4373bd74a91e091575ae: Log is configured to *not* fsync() on all Append() calls
00:10:53.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.611989 7586 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8e86a0326ccf4373bd74a91e091575ae: No bootstrap required, opened a new log
00:10:53.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.626368 7755 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c48b018d8a154bd8bd10fbf4739bcbc5: Bootstrap starting.
00:10:53.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.635258 7755 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P c48b018d8a154bd8bd10fbf4739bcbc5: Neither blocks nor log segments found. Creating new log.
00:10:53.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.639390 7755 log.cc:824] T 00000000000000000000000000000000 P c48b018d8a154bd8bd10fbf4739bcbc5: Log is configured to *not* fsync() on all Append() calls
00:10:53.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.641579 7586 raft_consensus.cc:348] T 00000000000000000000000000000000 P 8e86a0326ccf4373bd74a91e091575ae [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8e86a0326ccf4373bd74a91e091575ae" member_type: VOTER last_known_addr { host: "127.7.82.254" port: 41719 } } peers { permanent_uuid: "1f3c256f83ed423daef2312b243ba2d9" member_type: VOTER last_known_addr { host: "127.7.82.253" port: 34963 } } peers { permanent_uuid: "c48b018d8a154bd8bd10fbf4739bcbc5" member_type: VOTER last_known_addr { host: "127.7.82.252" port: 36925 } }
00:10:53.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.642699 7586 raft_consensus.cc:374] T 00000000000000000000000000000000 P 8e86a0326ccf4373bd74a91e091575ae [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
00:10:53.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.643015 7586 raft_consensus.cc:729] T 00000000000000000000000000000000 P 8e86a0326ccf4373bd74a91e091575ae [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8e86a0326ccf4373bd74a91e091575ae, State: Initialized, Role: FOLLOWER
00:10:53.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.643873 7586 consensus_queue.cc:260] T 00000000000000000000000000000000 P 8e86a0326ccf4373bd74a91e091575ae [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8e86a0326ccf4373bd74a91e091575ae" member_type: VOTER last_known_addr { host: "127.7.82.254" port: 41719 } } peers { permanent_uuid: "1f3c256f83ed423daef2312b243ba2d9" member_type: VOTER last_known_addr { host: "127.7.82.253" port: 34963 } } peers { permanent_uuid: "c48b018d8a154bd8bd10fbf4739bcbc5" member_type: VOTER last_known_addr { host: "127.7.82.252" port: 36925 } }
00:10:53.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.645366 7755 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c48b018d8a154bd8bd10fbf4739bcbc5: No bootstrap required, opened a new log
00:10:53.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.651125 7769 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8e86a0326ccf4373bd74a91e091575ae [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8e86a0326ccf4373bd74a91e091575ae" member_type: VOTER last_known_addr { host: "127.7.82.254" port: 41719 } } peers { permanent_uuid: "1f3c256f83ed423daef2312b243ba2d9" member_type: VOTER last_known_addr { host: "127.7.82.253" port: 34963 } } peers { permanent_uuid: "c48b018d8a154bd8bd10fbf4739bcbc5" member_type: VOTER last_known_addr { host: "127.7.82.252" port: 36925 } } }
00:10:53.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.651988 7769 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8e86a0326ccf4373bd74a91e091575ae [sys.catalog]: This master's current role is: FOLLOWER
00:10:53.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.652904 7586 sys_catalog.cc:564] T 00000000000000000000000000000000 P 8e86a0326ccf4373bd74a91e091575ae [sys.catalog]: configured and running, proceeding with master startup.
00:10:53.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.672828 7755 raft_consensus.cc:348] T 00000000000000000000000000000000 P c48b018d8a154bd8bd10fbf4739bcbc5 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8e86a0326ccf4373bd74a91e091575ae" member_type: VOTER last_known_addr { host: "127.7.82.254" port: 41719 } } peers { permanent_uuid: "1f3c256f83ed423daef2312b243ba2d9" member_type: VOTER last_known_addr { host: "127.7.82.253" port: 34963 } } peers { permanent_uuid: "c48b018d8a154bd8bd10fbf4739bcbc5" member_type: VOTER last_known_addr { host: "127.7.82.252" port: 36925 } }
00:10:53.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.674235 7755 raft_consensus.cc:374] T 00000000000000000000000000000000 P c48b018d8a154bd8bd10fbf4739bcbc5 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
00:10:53.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.674571 7755 raft_consensus.cc:729] T 00000000000000000000000000000000 P c48b018d8a154bd8bd10fbf4739bcbc5 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c48b018d8a154bd8bd10fbf4739bcbc5, State: Initialized, Role: FOLLOWER
00:10:53.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.675640 7755 consensus_queue.cc:260] T 00000000000000000000000000000000 P c48b018d8a154bd8bd10fbf4739bcbc5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8e86a0326ccf4373bd74a91e091575ae" member_type: VOTER last_known_addr { host: "127.7.82.254" port: 41719 } } peers { permanent_uuid: "1f3c256f83ed423daef2312b243ba2d9" member_type: VOTER last_known_addr { host: "127.7.82.253" port: 34963 } } peers { permanent_uuid: "c48b018d8a154bd8bd10fbf4739bcbc5" member_type: VOTER last_known_addr { host: "127.7.82.252" port: 36925 } }
00:10:53.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.679323 7781 sys_catalog.cc:455] T 00000000000000000000000000000000 P c48b018d8a154bd8bd10fbf4739bcbc5 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8e86a0326ccf4373bd74a91e091575ae" member_type: VOTER last_known_addr { host: "127.7.82.254" port: 41719 } } peers { permanent_uuid: "1f3c256f83ed423daef2312b243ba2d9" member_type: VOTER last_known_addr { host: "127.7.82.253" port: 34963 } } peers { permanent_uuid: "c48b018d8a154bd8bd10fbf4739bcbc5" member_type: VOTER last_known_addr { host: "127.7.82.252" port: 36925 } } }
00:10:53.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.680459 7781 sys_catalog.cc:458] T 00000000000000000000000000000000 P c48b018d8a154bd8bd10fbf4739bcbc5 [sys.catalog]: This master's current role is: FOLLOWER
00:10:53.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.681313 7755 sys_catalog.cc:564] T 00000000000000000000000000000000 P c48b018d8a154bd8bd10fbf4739bcbc5 [sys.catalog]: configured and running, proceeding with master startup.
00:10:53.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:53.686199 7780 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8e86a0326ccf4373bd74a91e091575ae: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:10:53.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:53.686461 7780 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:10:53.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:53.711452 7703 tablet.cc:2369] T 00000000000000000000000000000000 P c48b018d8a154bd8bd10fbf4739bcbc5: Can't schedule compaction. Clean time has not been advanced past its initial value.
00:10:53.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:53.728698 7795 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c48b018d8a154bd8bd10fbf4739bcbc5: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:10:53.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:53.729404 7795 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:10:53.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.685002 7515 master_runner.cc:186] Error getting master registration for 127.7.82.253:34963: OK, instance_id { permanent_uuid: "1f3c256f83ed423daef2312b243ba2d9" instance_seqno: 1786320651702692 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
00:10:53.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.738317 7515 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
00:10:53.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.733636 7669 master_runner.cc:186] Error getting master registration for 127.7.82.253:34963: OK, instance_id { permanent_uuid: "1f3c256f83ed423daef2312b243ba2d9" instance_seqno: 1786320651702692 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
00:10:53.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.767877 7669 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
00:10:53.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.840071 7769 raft_consensus.cc:482] T 00000000000000000000000000000000 P 8e86a0326ccf4373bd74a91e091575ae [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
00:10:53.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.840680 7769 raft_consensus.cc:504] T 00000000000000000000000000000000 P 8e86a0326ccf4373bd74a91e091575ae [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8e86a0326ccf4373bd74a91e091575ae" member_type: VOTER last_known_addr { host: "127.7.82.254" port: 41719 } } peers { permanent_uuid: "1f3c256f83ed423daef2312b243ba2d9" member_type: VOTER last_known_addr { host: "127.7.82.253" port: 34963 } } peers { permanent_uuid: "c48b018d8a154bd8bd10fbf4739bcbc5" member_type: VOTER last_known_addr { host: "127.7.82.252" port: 36925 } }
00:10:53.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.845155 7728 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "8e86a0326ccf4373bd74a91e091575ae" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c48b018d8a154bd8bd10fbf4739bcbc5" is_pre_election: true
00:10:53.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.849874 7728 raft_consensus.cc:2454] T 00000000000000000000000000000000 P c48b018d8a154bd8bd10fbf4739bcbc5 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8e86a0326ccf4373bd74a91e091575ae in term 0.
00:10:53.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.852083 7533 leader_election.cc:302] T 00000000000000000000000000000000 P 8e86a0326ccf4373bd74a91e091575ae [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 8e86a0326ccf4373bd74a91e091575ae, c48b018d8a154bd8bd10fbf4739bcbc5; no voters:
00:10:53.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.855104 7769 leader_election.cc:288] T 00000000000000000000000000000000 P 8e86a0326ccf4373bd74a91e091575ae [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1f3c256f83ed423daef2312b243ba2d9 (127.7.82.253:34963), c48b018d8a154bd8bd10fbf4739bcbc5 (127.7.82.252:36925)
00:10:53.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.856323 7769 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 8e86a0326ccf4373bd74a91e091575ae [term 0 FOLLOWER]: Leader pre-election won for term 1
00:10:53.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.857223 7769 raft_consensus.cc:482] T 00000000000000000000000000000000 P 8e86a0326ccf4373bd74a91e091575ae [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
00:10:53.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.858018 7769 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 8e86a0326ccf4373bd74a91e091575ae [term 0 FOLLOWER]: Advancing to term 1
00:10:53.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.867473 7769 raft_consensus.cc:504] T 00000000000000000000000000000000 P 8e86a0326ccf4373bd74a91e091575ae [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8e86a0326ccf4373bd74a91e091575ae" member_type: VOTER last_known_addr { host: "127.7.82.254" port: 41719 } } peers { permanent_uuid: "1f3c256f83ed423daef2312b243ba2d9" member_type: VOTER last_known_addr { host: "127.7.82.253" port: 34963 } } peers { permanent_uuid: "c48b018d8a154bd8bd10fbf4739bcbc5" member_type: VOTER last_known_addr { host: "127.7.82.252" port: 36925 } }
00:10:53.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.869400 7769 leader_election.cc:288] T 00000000000000000000000000000000 P 8e86a0326ccf4373bd74a91e091575ae [CANDIDATE]: Term 1 election: Requested vote from peers 1f3c256f83ed423daef2312b243ba2d9 (127.7.82.253:34963), c48b018d8a154bd8bd10fbf4739bcbc5 (127.7.82.252:36925)
00:10:53.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.870910 7728 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "8e86a0326ccf4373bd74a91e091575ae" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c48b018d8a154bd8bd10fbf4739bcbc5"
00:10:53.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.871490 7728 raft_consensus.cc:3037] T 00000000000000000000000000000000 P c48b018d8a154bd8bd10fbf4739bcbc5 [term 0 FOLLOWER]: Advancing to term 1
00:10:53.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.879962 7728 raft_consensus.cc:2454] T 00000000000000000000000000000000 P c48b018d8a154bd8bd10fbf4739bcbc5 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8e86a0326ccf4373bd74a91e091575ae in term 1.
00:10:53.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:53.882974 7538 tablet.cc:2369] T 00000000000000000000000000000000 P 8e86a0326ccf4373bd74a91e091575ae: Can't schedule compaction. Clean time has not been advanced past its initial value.
00:10:53.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.889353 7533 leader_election.cc:302] T 00000000000000000000000000000000 P 8e86a0326ccf4373bd74a91e091575ae [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 8e86a0326ccf4373bd74a91e091575ae, c48b018d8a154bd8bd10fbf4739bcbc5; no voters:
00:10:53.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.896638 7769 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 8e86a0326ccf4373bd74a91e091575ae [term 1 FOLLOWER]: Leader election won for term 1
00:10:53.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.897127 7769 raft_consensus.cc:686] T 00000000000000000000000000000000 P 8e86a0326ccf4373bd74a91e091575ae [term 1 LEADER]: Becoming Leader. State: Replica: 8e86a0326ccf4373bd74a91e091575ae, State: Running, Role: LEADER
00:10:53.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.898046 7769 consensus_queue.cc:237] T 00000000000000000000000000000000 P 8e86a0326ccf4373bd74a91e091575ae [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8e86a0326ccf4373bd74a91e091575ae" member_type: VOTER last_known_addr { host: "127.7.82.254" port: 41719 } } peers { permanent_uuid: "1f3c256f83ed423daef2312b243ba2d9" member_type: VOTER last_known_addr { host: "127.7.82.253" port: 34963 } } peers { permanent_uuid: "c48b018d8a154bd8bd10fbf4739bcbc5" member_type: VOTER last_known_addr { host: "127.7.82.252" port: 36925 } }
00:10:53.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.904088 7797 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8e86a0326ccf4373bd74a91e091575ae [sys.catalog]: SysCatalogTable state changed. Reason: New leader 8e86a0326ccf4373bd74a91e091575ae. Latest consensus state: current_term: 1 leader_uuid: "8e86a0326ccf4373bd74a91e091575ae" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8e86a0326ccf4373bd74a91e091575ae" member_type: VOTER last_known_addr { host: "127.7.82.254" port: 41719 } } peers { permanent_uuid: "1f3c256f83ed423daef2312b243ba2d9" member_type: VOTER last_known_addr { host: "127.7.82.253" port: 34963 } } peers { permanent_uuid: "c48b018d8a154bd8bd10fbf4739bcbc5" member_type: VOTER last_known_addr { host: "127.7.82.252" port: 36925 } } }
00:10:53.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.905236 7797 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8e86a0326ccf4373bd74a91e091575ae [sys.catalog]: This master's current role is: LEADER
00:10:53.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.917764 7799 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
00:10:53.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.931161 7799 catalog_manager.cc:1484] Initializing Kudu cluster ID...
00:10:53.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.954243 7728 raft_consensus.cc:1261] T 00000000000000000000000000000000 P c48b018d8a154bd8bd10fbf4739bcbc5 [term 1 FOLLOWER]: Refusing update from remote peer 8e86a0326ccf4373bd74a91e091575ae: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
00:10:53.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:53.957212 7769 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 8e86a0326ccf4373bd74a91e091575ae [LEADER]: Connected to new peer: Peer: permanent_uuid: "c48b018d8a154bd8bd10fbf4739bcbc5" member_type: VOTER last_known_addr { host: "127.7.82.252" port: 36925 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
00:10:54.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:54.000813 7781 sys_catalog.cc:455] T 00000000000000000000000000000000 P c48b018d8a154bd8bd10fbf4739bcbc5 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 8e86a0326ccf4373bd74a91e091575ae. Latest consensus state: current_term: 1 leader_uuid: "8e86a0326ccf4373bd74a91e091575ae" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8e86a0326ccf4373bd74a91e091575ae" member_type: VOTER last_known_addr { host: "127.7.82.254" port: 41719 } } peers { permanent_uuid: "1f3c256f83ed423daef2312b243ba2d9" member_type: VOTER last_known_addr { host: "127.7.82.253" port: 34963 } } peers { permanent_uuid: "c48b018d8a154bd8bd10fbf4739bcbc5" member_type: VOTER last_known_addr { host: "127.7.82.252" port: 36925 } } }
00:10:54.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:54.001535 7781 sys_catalog.cc:458] T 00000000000000000000000000000000 P c48b018d8a154bd8bd10fbf4739bcbc5 [sys.catalog]: This master's current role is: FOLLOWER
00:10:54.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:54.032205 7781 sys_catalog.cc:455] T 00000000000000000000000000000000 P c48b018d8a154bd8bd10fbf4739bcbc5 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "8e86a0326ccf4373bd74a91e091575ae" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8e86a0326ccf4373bd74a91e091575ae" member_type: VOTER last_known_addr { host: "127.7.82.254" port: 41719 } } peers { permanent_uuid: "1f3c256f83ed423daef2312b243ba2d9" member_type: VOTER last_known_addr { host: "127.7.82.253" port: 34963 } } peers { permanent_uuid: "c48b018d8a154bd8bd10fbf4739bcbc5" member_type: VOTER last_known_addr { host: "127.7.82.252" port: 36925 } } }
00:10:54.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:54.033004 7781 sys_catalog.cc:458] T 00000000000000000000000000000000 P c48b018d8a154bd8bd10fbf4739bcbc5 [sys.catalog]: This master's current role is: FOLLOWER
00:10:54.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:54.044646 7797 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8e86a0326ccf4373bd74a91e091575ae [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "8e86a0326ccf4373bd74a91e091575ae" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8e86a0326ccf4373bd74a91e091575ae" member_type: VOTER last_known_addr { host: "127.7.82.254" port: 41719 } } peers { permanent_uuid: "1f3c256f83ed423daef2312b243ba2d9" member_type: VOTER last_known_addr { host: "127.7.82.253" port: 34963 } } peers { permanent_uuid: "c48b018d8a154bd8bd10fbf4739bcbc5" member_type: VOTER last_known_addr { host: "127.7.82.252" port: 36925 } } }
00:10:54.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:54.045390 7797 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8e86a0326ccf4373bd74a91e091575ae [sys.catalog]: This master's current role is: LEADER
00:10:54.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:54.052170 7799 catalog_manager.cc:1347] Generated new cluster ID: 3590803e6ea74ea397025da106d5bf9b
00:10:54.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:54.052459 7799 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
00:10:54.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:54.081022 7667 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.82.252" port: 36925 } attempt: 7
00:10:54.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:54.117828 7799 catalog_manager.cc:1370] Generated new certificate authority record
00:10:54.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:54.130771 7667 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1f3c256f83ed423daef2312b243ba2d9: Bootstrap starting.
00:10:54.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:54.126716 7799 catalog_manager.cc:1504] Loading token signing keys...
00:10:54.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:54.139531 7667 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 1f3c256f83ed423daef2312b243ba2d9: Neither blocks nor log segments found. Creating new log.
00:10:54.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:54.142149 7667 log.cc:824] T 00000000000000000000000000000000 P 1f3c256f83ed423daef2312b243ba2d9: Log is configured to *not* fsync() on all Append() calls
00:10:54.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:54.166970 7667 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1f3c256f83ed423daef2312b243ba2d9: No bootstrap required, opened a new log
00:10:54.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:54.177623 7799 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 8e86a0326ccf4373bd74a91e091575ae: Generated new TSK 0
00:10:54.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:54.178840 7799 catalog_manager.cc:1514] Initializing in-progress tserver states...
00:10:54.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:54.205569 7499 external_mini_cluster.cc:1326] Running /tmp/dist-test-task3KSwpE/build/tsan/bin/kudu
00:10:54.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task3KSwpE/build/tsan/bin/kudu
00:10:54.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/ts-0/wal
00:10:54.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/ts-0/data
00:10:54.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:10:54.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:10:54.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:10:54.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:10:54.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:10:54.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:10:54.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/ts-0/logs
00:10:54.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/ts-0/data/info.pb
00:10:54.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:10:54.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:10:54.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:10:54.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:10:54.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:10:54.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:10:54.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:10:54.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:10:54.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:10:54.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:10:54.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.82.193:0
00:10:54.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.7.82.193
00:10:54.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.82.193
00:10:54.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:10:54.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.7.82.254:41719,127.7.82.253:34963,127.7.82.252:36925
00:10:54.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.82.212:46327
00:10:54.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:10:54.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:10:54.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:10:54.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:54.210827 7667 raft_consensus.cc:348] T 00000000000000000000000000000000 P 1f3c256f83ed423daef2312b243ba2d9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8e86a0326ccf4373bd74a91e091575ae" member_type: VOTER last_known_addr { host: "127.7.82.254" port: 41719 } } peers { permanent_uuid: "1f3c256f83ed423daef2312b243ba2d9" member_type: VOTER last_known_addr { host: "127.7.82.253" port: 34963 } } peers { permanent_uuid: "c48b018d8a154bd8bd10fbf4739bcbc5" member_type: VOTER last_known_addr { host: "127.7.82.252" port: 36925 } }
00:10:54.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:54.211928 7667 raft_consensus.cc:374] T 00000000000000000000000000000000 P 1f3c256f83ed423daef2312b243ba2d9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
00:10:54.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:54.212427 7667 raft_consensus.cc:729] T 00000000000000000000000000000000 P 1f3c256f83ed423daef2312b243ba2d9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1f3c256f83ed423daef2312b243ba2d9, State: Initialized, Role: FOLLOWER
00:10:54.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:54.213655 7667 consensus_queue.cc:260] T 00000000000000000000000000000000 P 1f3c256f83ed423daef2312b243ba2d9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8e86a0326ccf4373bd74a91e091575ae" member_type: VOTER last_known_addr { host: "127.7.82.254" port: 41719 } } peers { permanent_uuid: "1f3c256f83ed423daef2312b243ba2d9" member_type: VOTER last_known_addr { host: "127.7.82.253" port: 34963 } } peers { permanent_uuid: "c48b018d8a154bd8bd10fbf4739bcbc5" member_type: VOTER last_known_addr { host: "127.7.82.252" port: 36925 } }
00:10:54.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:54.219331 7809 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1f3c256f83ed423daef2312b243ba2d9 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8e86a0326ccf4373bd74a91e091575ae" member_type: VOTER last_known_addr { host: "127.7.82.254" port: 41719 } } peers { permanent_uuid: "1f3c256f83ed423daef2312b243ba2d9" member_type: VOTER last_known_addr { host: "127.7.82.253" port: 34963 } } peers { permanent_uuid: "c48b018d8a154bd8bd10fbf4739bcbc5" member_type: VOTER last_known_addr { host: "127.7.82.252" port: 36925 } } }
00:10:54.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:54.225929 7809 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1f3c256f83ed423daef2312b243ba2d9 [sys.catalog]: This master's current role is: FOLLOWER
00:10:54.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:54.230499 7638 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "8e86a0326ccf4373bd74a91e091575ae" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1f3c256f83ed423daef2312b243ba2d9" is_pre_election: true
00:10:54.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:54.231544 7638 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 1f3c256f83ed423daef2312b243ba2d9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8e86a0326ccf4373bd74a91e091575ae in term 0.
00:10:54.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:54.233312 7636 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 1f3c256f83ed423daef2312b243ba2d9 [term 0 FOLLOWER]: Advancing to term 1
00:10:54.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:54.240406 7637 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "8e86a0326ccf4373bd74a91e091575ae" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1f3c256f83ed423daef2312b243ba2d9"
00:10:54.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:54.242815 7636 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 1f3c256f83ed423daef2312b243ba2d9 [term 1 FOLLOWER]: Refusing update from remote peer 8e86a0326ccf4373bd74a91e091575ae: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
00:10:54.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:54.243853 7667 sys_catalog.cc:564] T 00000000000000000000000000000000 P 1f3c256f83ed423daef2312b243ba2d9 [sys.catalog]: configured and running, proceeding with master startup.
00:10:54.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:54.244515 7769 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 8e86a0326ccf4373bd74a91e091575ae [LEADER]: Connected to new peer: Peer: permanent_uuid: "1f3c256f83ed423daef2312b243ba2d9" member_type: VOTER last_known_addr { host: "127.7.82.253" port: 34963 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
00:10:54.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:54.324939 7819 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1f3c256f83ed423daef2312b243ba2d9 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "8e86a0326ccf4373bd74a91e091575ae" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8e86a0326ccf4373bd74a91e091575ae" member_type: VOTER last_known_addr { host: "127.7.82.254" port: 41719 } } peers { permanent_uuid: "1f3c256f83ed423daef2312b243ba2d9" member_type: VOTER last_known_addr { host: "127.7.82.253" port: 34963 } } peers { permanent_uuid: "c48b018d8a154bd8bd10fbf4739bcbc5" member_type: VOTER last_known_addr { host: "127.7.82.252" port: 36925 } } }
00:10:54.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:54.326933 7819 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1f3c256f83ed423daef2312b243ba2d9 [sys.catalog]: This master's current role is: FOLLOWER
00:10:54.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:54.323873 7809 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1f3c256f83ed423daef2312b243ba2d9 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 8e86a0326ccf4373bd74a91e091575ae. Latest consensus state: current_term: 1 leader_uuid: "8e86a0326ccf4373bd74a91e091575ae" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8e86a0326ccf4373bd74a91e091575ae" member_type: VOTER last_known_addr { host: "127.7.82.254" port: 41719 } } peers { permanent_uuid: "1f3c256f83ed423daef2312b243ba2d9" member_type: VOTER last_known_addr { host: "127.7.82.253" port: 34963 } } peers { permanent_uuid: "c48b018d8a154bd8bd10fbf4739bcbc5" member_type: VOTER last_known_addr { host: "127.7.82.252" port: 36925 } } }
00:10:54.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:54.327999 7809 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1f3c256f83ed423daef2312b243ba2d9 [sys.catalog]: This master's current role is: FOLLOWER
00:10:54.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:54.340189 7825 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
00:10:54.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:54.719157 7807 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
00:10:54.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:54.719851 7807 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:10:54.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:54.720661 7807 flags.cc:425] Enabled unsafe flag: --never_fsync=true
00:10:54.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:54.741613 7795 catalog_manager.cc:1259] Loaded cluster ID: 3590803e6ea74ea397025da106d5bf9b
00:10:54.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:54.742226 7795 catalog_manager.cc:1552] T 00000000000000000000000000000000 P c48b018d8a154bd8bd10fbf4739bcbc5: loading cluster ID for follower catalog manager: success
00:10:54.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:54.749889 7795 catalog_manager.cc:1574] T 00000000000000000000000000000000 P c48b018d8a154bd8bd10fbf4739bcbc5: acquiring CA information for follower catalog manager: success
00:10:54.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:54.769611 7795 catalog_manager.cc:1602] T 00000000000000000000000000000000 P c48b018d8a154bd8bd10fbf4739bcbc5: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
00:10:54.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:54.776121 7807 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
00:10:54.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:54.777782 7807 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.82.193
00:10:54.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:54.838735 7807 tablet_server_runner.cc:78] Tablet server non-default flags:
00:10:54.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:10:54.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.82.212:46327
00:10:54.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:10:54.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:10:54.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/ts-0/data
00:10:54.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/ts-0/wal
00:10:54.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:10:54.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:10:54.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.82.193:0
00:10:54.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:10:54.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:10:54.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:10:54.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/ts-0/data/info.pb
00:10:54.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.82.193
00:10:54.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:10:54.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.7.82.254:41719,127.7.82.253:34963,127.7.82.252:36925
00:10:54.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:10:54.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:10:54.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:10:54.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:10:54.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:10:54.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.7.82.193
00:10:54.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/ts-0/logs
00:10:54.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:10:54.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:10:54.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
00:10:54.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:10:54.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
00:10:54.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
00:10:54.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:10:54.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 23:15:58 UTC on ccycloud.quasar-ddmxrv.root.comops.site
00:10:54.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4576820
00:10:54.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
00:10:54.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:54.840351 7807 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:10:54.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:54.842468 7807 file_cache.cc:493] Constructed file cache file cache with capacity 419430
00:10:54.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:54.866915 7834 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:10:54.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:54.873144 7769 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8e86a0326ccf4373bd74a91e091575ae [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "8e86a0326ccf4373bd74a91e091575ae" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8e86a0326ccf4373bd74a91e091575ae" member_type: VOTER last_known_addr { host: "127.7.82.254" port: 41719 } } peers { permanent_uuid: "1f3c256f83ed423daef2312b243ba2d9" member_type: VOTER last_known_addr { host: "127.7.82.253" port: 34963 } } peers { permanent_uuid: "c48b018d8a154bd8bd10fbf4739bcbc5" member_type: VOTER last_known_addr { host: "127.7.82.252" port: 36925 } } }
00:10:54.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:54.875475 7769 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8e86a0326ccf4373bd74a91e091575ae [sys.catalog]: This master's current role is: LEADER
00:10:55.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:55.361735 7825 catalog_manager.cc:1259] Loaded cluster ID: 3590803e6ea74ea397025da106d5bf9b
00:10:55.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:55.366298 7825 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 1f3c256f83ed423daef2312b243ba2d9: loading cluster ID for follower catalog manager: success
00:10:55.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:55.400789 7825 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 1f3c256f83ed423daef2312b243ba2d9: acquiring CA information for follower catalog manager: success
00:10:55.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:55.410158 7825 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 1f3c256f83ed423daef2312b243ba2d9: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
00:10:56.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:56.260116 7832 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 7807
00:10:56.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:56.440327 7832 kernel_stack_watchdog.cc:198] Thread 7807 stuck at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:637 for 391ms:
00:10:56.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
00:10:56.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
00:10:56.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
00:10:56.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
00:10:56.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:54.871439 7835 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:10:56.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:56.450670 7807 thread.cc:636] GCE (cloud detector) Time spent creating pthread: real 1.584s user 0.545s sys 0.845s
00:10:56.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:56.451054 7807 thread.cc:602] GCE (cloud detector) Time spent starting thread: real 1.584s user 0.545s sys 0.845s
00:10:56.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:56.461279 7844 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:10:56.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:56.461828 7807 server_base.cc:1034] running on GCE node
00:10:56.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:56.463212 7807 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:10:56.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:56.470806 7807 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:10:56.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:56.475215 7807 hybrid_clock.cc:648] HybridClock initialized: now 1786320656475165 us; error 47 us; skew 500 ppm
00:10:56.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:56.476306 7807 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:10:56.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:56.501118 7807 webserver.cc:460] Webserver started at http://127.7.82.193:45747/ using document root <none> and password file <none>
00:10:56.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:56.502607 7807 fs_manager.cc:359] Metadata directory not provided
00:10:56.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:56.502925 7807 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:10:56.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:56.503535 7807 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
00:10:56.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:56.511634 7807 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/ts-0/data/instance:
00:10:56.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8378e809c3eb42589ce97c5823c66c74"
00:10:56.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-10 00:10:56 on dist-test-slave-cbbz"
00:10:56.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:56.513538 7807 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/ts-0/wal/instance:
00:10:56.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8378e809c3eb42589ce97c5823c66c74"
00:10:56.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-10 00:10:56 on dist-test-slave-cbbz"
00:10:56.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:56.530086 7807 fs_manager.cc:691] Time spent creating directory manager: real 0.015s user 0.010s sys 0.006s
00:10:56.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:56.544062 7849 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
00:10:56.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:56.545517 7807 fs_manager.cc:725] Time spent opening block manager: real 0.010s user 0.006s sys 0.005s
00:10:56.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:56.545948 7807 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/ts-0/data,/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/ts-0/wal
00:10:56.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8378e809c3eb42589ce97c5823c66c74"
00:10:56.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-10 00:10:56 on dist-test-slave-cbbz"
00:10:56.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:56.546459 7807 fs_report.cc:383] FS layout report
00:10:56.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:10:56.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/ts-0/wal
00:10:56.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/ts-0/wal
00:10:56.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/ts-0/data/data
00:10:56.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:10:56.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:10:56.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:10:56.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:10:56.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:10:56.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:10:56.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:10:56.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:10:56.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:10:56.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:10:56.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:10:56.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:56.627815 7807 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
00:10:56.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:56.630373 7807 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:10:56.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:56.631279 7807 kserver.cc:163] Server-wide thread pool size limit: 3276
00:10:56.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:56.641094 7807 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
00:10:56.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:56.653757 7807 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
00:10:56.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:56.654381 7807 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.001s user 0.001s sys 0.000s
00:10:56.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:56.654910 7807 ts_tablet_manager.cc:610] Registered 0 tablets
00:10:56.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:56.655320 7807 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
00:10:57.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:57.242522 7807 rpc_server.cc:307] RPC server started. Bound to: 127.7.82.193:40489
00:10:57.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:57.242676 7977 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.82.193:40489 every 8 connection(s)
00:10:57.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:57.247023 7807 server_base.cc:1166] Dumped server information to /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/ts-0/data/info.pb
00:10:57.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:57.256788 7499 external_mini_cluster.cc:1388] Started /tmp/dist-test-task3KSwpE/build/tsan/bin/kudu as pid 7807
00:10:57.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:57.257247 7499 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/ts-0/wal/instance
00:10:57.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:57.275542 7499 external_mini_cluster.cc:1326] Running /tmp/dist-test-task3KSwpE/build/tsan/bin/kudu
00:10:57.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task3KSwpE/build/tsan/bin/kudu
00:10:57.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/ts-1/wal
00:10:57.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/ts-1/data
00:10:57.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:10:57.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:10:57.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:10:57.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:10:57.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:10:57.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:10:57.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/ts-1/logs
00:10:57.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/ts-1/data/info.pb
00:10:57.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:10:57.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:10:57.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:10:57.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:10:57.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:10:57.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:10:57.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:10:57.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:10:57.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:10:57.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:10:57.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.82.194:0
00:10:57.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.7.82.194
00:10:57.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.82.194
00:10:57.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:10:57.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.7.82.254:41719,127.7.82.253:34963,127.7.82.252:36925
00:10:57.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.82.212:46327
00:10:57.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:10:57.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:10:57.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:10:57.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:57.315181 7982 heartbeater.cc:347] Connected to a master server at 127.7.82.254:41719
00:10:57.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:57.315737 7982 heartbeater.cc:464] Registering TS with master...
00:10:57.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:57.317013 7982 heartbeater.cc:511] Master 127.7.82.254:41719 requested a full tablet report, sending...
00:10:57.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:57.321398 7549 ts_manager.cc:194] Registered new tserver with Master: 8378e809c3eb42589ce97c5823c66c74 (127.7.82.193:40489)
00:10:57.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:57.324577 7549 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.82.193:44155
00:10:57.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:57.335865 7978 heartbeater.cc:347] Connected to a master server at 127.7.82.253:34963
00:10:57.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:57.336244 7978 heartbeater.cc:464] Registering TS with master...
00:10:57.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:57.336979 7978 heartbeater.cc:511] Master 127.7.82.253:34963 requested a full tablet report, sending...
00:10:57.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:57.340010 7628 ts_manager.cc:194] Registered new tserver with Master: 8378e809c3eb42589ce97c5823c66c74 (127.7.82.193:40489)
00:10:57.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:57.343204 7979 heartbeater.cc:347] Connected to a master server at 127.7.82.252:36925
00:10:57.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:57.343546 7979 heartbeater.cc:464] Registering TS with master...
00:10:57.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:57.344280 7979 heartbeater.cc:511] Master 127.7.82.252:36925 requested a full tablet report, sending...
00:10:57.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:57.347128 7717 ts_manager.cc:194] Registered new tserver with Master: 8378e809c3eb42589ce97c5823c66c74 (127.7.82.193:40489)
00:10:57.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:57.723663 7988 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
00:10:57.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:57.724342 7988 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:10:57.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:57.725131 7988 flags.cc:425] Enabled unsafe flag: --never_fsync=true
00:10:57.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:57.771394 7988 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
00:10:57.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:57.772491 7988 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.82.194
00:10:57.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:57.810729 7988 tablet_server_runner.cc:78] Tablet server non-default flags:
00:10:57.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:10:57.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.82.212:46327
00:10:57.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:10:57.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:10:57.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/ts-1/data
00:10:57.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/ts-1/wal
00:10:57.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:10:57.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:10:57.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.82.194:0
00:10:57.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:10:57.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:10:57.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:10:57.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/ts-1/data/info.pb
00:10:57.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.82.194
00:10:57.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:10:57.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.7.82.254:41719,127.7.82.253:34963,127.7.82.252:36925
00:10:57.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:10:57.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:10:57.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:10:57.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:10:57.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:10:57.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.7.82.194
00:10:57.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/ts-1/logs
00:10:57.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:10:57.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:10:57.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
00:10:57.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:10:57.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
00:10:57.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
00:10:57.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:10:57.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 23:15:58 UTC on ccycloud.quasar-ddmxrv.root.comops.site
00:10:57.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4576820
00:10:57.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
00:10:57.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:57.812363 7988 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:10:57.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:57.814515 7988 file_cache.cc:493] Constructed file cache file cache with capacity 419430
00:10:57.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:57.835269 7996 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:10:58.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:58.330590 7982 heartbeater.cc:503] Master 127.7.82.254:41719 was elected leader, sending a full tablet report...
00:10:59.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:57.835894 7998 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:10:59.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:57.837297 7995 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:10:59.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:59.046319 7997 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
00:10:59.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:59.046602 7988 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
00:10:59.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:59.050084 7988 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:10:59.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:59.052801 7988 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:10:59.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:59.054224 7988 hybrid_clock.cc:648] HybridClock initialized: now 1786320659054188 us; error 51 us; skew 500 ppm
00:10:59.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:59.054978 7988 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:10:59.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:59.061051 7988 webserver.cc:460] Webserver started at http://127.7.82.194:46397/ using document root <none> and password file <none>
00:10:59.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:59.061925 7988 fs_manager.cc:359] Metadata directory not provided
00:10:59.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:59.062108 7988 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:10:59.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:59.062508 7988 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
00:10:59.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:59.067296 7988 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/ts-1/data/instance:
00:10:59.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "463e66937ec74d77a1ffc7cdf3471cb1"
00:10:59.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-10 00:10:59 on dist-test-slave-cbbz"
00:10:59.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:59.068531 7988 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/ts-1/wal/instance:
00:10:59.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "463e66937ec74d77a1ffc7cdf3471cb1"
00:10:59.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-10 00:10:59 on dist-test-slave-cbbz"
00:10:59.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:59.075809 7988 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
00:10:59.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:59.081394 8008 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
00:10:59.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:59.082554 7988 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
00:10:59.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:59.082845 7988 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/ts-1/data,/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/ts-1/wal
00:10:59.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "463e66937ec74d77a1ffc7cdf3471cb1"
00:10:59.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-10 00:10:59 on dist-test-slave-cbbz"
00:10:59.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:59.083143 7988 fs_report.cc:383] FS layout report
00:10:59.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:10:59.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/ts-1/wal
00:10:59.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/ts-1/wal
00:10:59.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/ts-1/data/data
00:10:59.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:10:59.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:10:59.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:10:59.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:10:59.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:10:59.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:10:59.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:10:59.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:10:59.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:10:59.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:10:59.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:10:59.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:59.125094 7988 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
00:10:59.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:59.126451 7988 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:10:59.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:59.126859 7988 kserver.cc:163] Server-wide thread pool size limit: 3276
00:10:59.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:59.129379 7988 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
00:10:59.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:59.133569 7988 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
00:10:59.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:59.133800 7988 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
00:10:59.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:59.134013 7988 ts_tablet_manager.cc:610] Registered 0 tablets
00:10:59.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:59.134138 7988 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
00:10:59.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:59.282902 7988 rpc_server.cc:307] RPC server started. Bound to: 127.7.82.194:33631
00:10:59.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:59.283156 8128 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.82.194:33631 every 8 connection(s)
00:10:59.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:59.285431 7988 server_base.cc:1166] Dumped server information to /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/ts-1/data/info.pb
00:10:59.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:59.291056 7499 external_mini_cluster.cc:1388] Started /tmp/dist-test-task3KSwpE/build/tsan/bin/kudu as pid 7988
00:10:59.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:59.291888 7499 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/ts-1/wal/instance
00:10:59.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:59.310389 7499 external_mini_cluster.cc:1326] Running /tmp/dist-test-task3KSwpE/build/tsan/bin/kudu
00:10:59.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task3KSwpE/build/tsan/bin/kudu
00:10:59.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/ts-2/wal
00:10:59.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/ts-2/data
00:10:59.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:10:59.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:10:59.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:10:59.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:10:59.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:10:59.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:10:59.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/ts-2/logs
00:10:59.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/ts-2/data/info.pb
00:10:59.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:10:59.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:10:59.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:10:59.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:10:59.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:10:59.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:10:59.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:10:59.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:10:59.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:10:59.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:10:59.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.82.195:0
00:10:59.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.7.82.195
00:10:59.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.82.195
00:10:59.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:10:59.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.7.82.254:41719,127.7.82.253:34963,127.7.82.252:36925
00:10:59.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.82.212:46327
00:10:59.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:10:59.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:10:59.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:10:59.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:59.332345 8131 heartbeater.cc:347] Connected to a master server at 127.7.82.252:36925
00:10:59.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:59.332829 8131 heartbeater.cc:464] Registering TS with master...
00:10:59.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:59.334072 8131 heartbeater.cc:511] Master 127.7.82.252:36925 requested a full tablet report, sending...
00:10:59.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:59.334502 8132 heartbeater.cc:347] Connected to a master server at 127.7.82.254:41719
00:10:59.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:59.334904 8132 heartbeater.cc:464] Registering TS with master...
00:10:59.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:59.335846 8132 heartbeater.cc:511] Master 127.7.82.254:41719 requested a full tablet report, sending...
00:10:59.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:59.337634 7717 ts_manager.cc:194] Registered new tserver with Master: 463e66937ec74d77a1ffc7cdf3471cb1 (127.7.82.194:33631)
00:10:59.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:59.338361 7548 ts_manager.cc:194] Registered new tserver with Master: 463e66937ec74d77a1ffc7cdf3471cb1 (127.7.82.194:33631)
00:10:59.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:59.340597 8130 heartbeater.cc:347] Connected to a master server at 127.7.82.253:34963
00:10:59.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:59.340942 8130 heartbeater.cc:464] Registering TS with master...
00:10:59.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:59.340859 7548 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.82.194:52991
00:10:59.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:59.341789 8130 heartbeater.cc:511] Master 127.7.82.253:34963 requested a full tablet report, sending...
00:10:59.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:59.343972 7628 ts_manager.cc:194] Registered new tserver with Master: 463e66937ec74d77a1ffc7cdf3471cb1 (127.7.82.194:33631)
00:10:59.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:59.648595 8140 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
00:10:59.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:59.649075 8140 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:10:59.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:59.649507 8140 flags.cc:425] Enabled unsafe flag: --never_fsync=true
00:10:59.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:59.680341 8140 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
00:10:59.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:59.681126 8140 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.82.195
00:10:59.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:59.715142 8140 tablet_server_runner.cc:78] Tablet server non-default flags:
00:10:59.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:10:59.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.82.212:46327
00:10:59.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:10:59.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:10:59.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/ts-2/data
00:10:59.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/ts-2/wal
00:10:59.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:10:59.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:10:59.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.82.195:0
00:10:59.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:10:59.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:10:59.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:10:59.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/ts-2/data/info.pb
00:10:59.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.82.195
00:10:59.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:10:59.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.7.82.254:41719,127.7.82.253:34963,127.7.82.252:36925
00:10:59.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:10:59.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:10:59.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:10:59.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:10:59.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:10:59.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.7.82.195
00:10:59.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/ts-2/logs
00:10:59.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:10:59.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:10:59.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
00:10:59.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:10:59.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
00:10:59.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
00:10:59.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:10:59.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 23:15:58 UTC on ccycloud.quasar-ddmxrv.root.comops.site
00:10:59.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4576820
00:10:59.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
00:10:59.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:59.716477 8140 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:10:59.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:59.718262 8140 file_cache.cc:493] Constructed file cache file cache with capacity 419430
00:10:59.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:59.740259 8148 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:11:00.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:00.345640 8132 heartbeater.cc:503] Master 127.7.82.254:41719 was elected leader, sending a full tablet report...
00:11:00.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:59.740497 8146 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:11:00.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:10:59.740983 8150 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:11:00.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:10:59.741596 8140 server_base.cc:1034] running on GCE node
00:11:00.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:00.942625 8140 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:11:00.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:00.945379 8140 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:11:00.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:00.946841 8140 hybrid_clock.cc:648] HybridClock initialized: now 1786320660946791 us; error 47 us; skew 500 ppm
00:11:00.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:00.947708 8140 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:11:00.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:00.954737 8140 webserver.cc:460] Webserver started at http://127.7.82.195:40953/ using document root <none> and password file <none>
00:11:00.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:00.955696 8140 fs_manager.cc:359] Metadata directory not provided
00:11:00.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:00.955941 8140 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:11:00.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:00.956494 8140 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
00:11:00.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:00.961480 8140 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/ts-2/data/instance:
00:11:00.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "23c451930dfe4fc3af4e845c85859252"
00:11:00.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-10 00:11:00 on dist-test-slave-cbbz"
00:11:00.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:00.962767 8140 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/ts-2/wal/instance:
00:11:00.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "23c451930dfe4fc3af4e845c85859252"
00:11:00.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-10 00:11:00 on dist-test-slave-cbbz"
00:11:00.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:00.970423 8140 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
00:11:00.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:00.976204 8162 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
00:11:00.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:00.977203 8140 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
00:11:00.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:00.977557 8140 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/ts-2/data,/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/ts-2/wal
00:11:00.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "23c451930dfe4fc3af4e845c85859252"
00:11:00.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-10 00:11:00 on dist-test-slave-cbbz"
00:11:00.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:00.977905 8140 fs_report.cc:383] FS layout report
00:11:00.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:11:00.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/ts-2/wal
00:11:00.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/ts-2/wal
00:11:00.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/ts-2/data/data
00:11:00.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:11:00.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:11:00.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:11:00.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:11:00.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:11:00.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:11:00.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:11:00.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:11:00.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:11:00.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:11:00.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:11:01.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:01.018561 8140 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
00:11:01.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:01.020570 8140 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:11:01.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:01.021085 8140 kserver.cc:163] Server-wide thread pool size limit: 3276
00:11:01.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:01.023648 8140 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
00:11:01.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:01.027873 8140 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
00:11:01.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:01.028136 8140 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
00:11:01.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:01.028496 8140 ts_tablet_manager.cc:610] Registered 0 tablets
00:11:01.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:01.028685 8140 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
00:11:01.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:01.175554 8140 rpc_server.cc:307] RPC server started. Bound to: 127.7.82.195:37997
00:11:01.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:01.175712 8281 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.82.195:37997 every 8 connection(s)
00:11:01.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:01.178265 8140 server_base.cc:1166] Dumped server information to /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/ts-2/data/info.pb
00:11:01.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:01.181275 7499 external_mini_cluster.cc:1388] Started /tmp/dist-test-task3KSwpE/build/tsan/bin/kudu as pid 8140
00:11:01.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:01.181865 7499 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6588517925771719438/ts-2/wal/instance
00:11:01.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:01.222579 8282 heartbeater.cc:347] Connected to a master server at 127.7.82.253:34963
00:11:01.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:01.222837 8283 heartbeater.cc:347] Connected to a master server at 127.7.82.252:36925
00:11:01.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:01.223116 8282 heartbeater.cc:464] Registering TS with master...
00:11:01.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:01.222847 8284 heartbeater.cc:347] Connected to a master server at 127.7.82.254:41719
00:11:01.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:01.223305 8283 heartbeater.cc:464] Registering TS with master...
00:11:01.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:01.223692 8284 heartbeater.cc:464] Registering TS with master...
00:11:01.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:01.224548 8282 heartbeater.cc:511] Master 127.7.82.253:34963 requested a full tablet report, sending...
00:11:01.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:01.224704 8283 heartbeater.cc:511] Master 127.7.82.252:36925 requested a full tablet report, sending...
00:11:01.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:01.225245 8284 heartbeater.cc:511] Master 127.7.82.254:41719 requested a full tablet report, sending...
00:11:01.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:01.227726 7549 ts_manager.cc:194] Registered new tserver with Master: 23c451930dfe4fc3af4e845c85859252 (127.7.82.195:37997)
00:11:01.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:01.227797 7627 ts_manager.cc:194] Registered new tserver with Master: 23c451930dfe4fc3af4e845c85859252 (127.7.82.195:37997)
00:11:01.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:01.228129 7715 ts_manager.cc:194] Registered new tserver with Master: 23c451930dfe4fc3af4e845c85859252 (127.7.82.195:37997)
00:11:01.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:01.229611 7549 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.82.195:49041
00:11:01.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:01.250874 7499 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
00:11:01.263 [DEBUG - main] (MiniKuduCluster.java:195) Response:
00:11:01.265 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
00:11:01.276 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.7.82.254"
port: 41719
}
bound_http_address {
host: "127.7.82.254"
port: 45777
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.7.82.253"
port: 34963
}
bound_http_address {
host: "127.7.82.253"
port: 37189
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.7.82.252"
port: 36925
}
bound_http_address {
host: "127.7.82.252"
port: 38247
}
}
}
00:11:01.283 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
00:11:01.286 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.7.82.193"
port: 40489
}
bound_http_address {
host: "127.7.82.193"
port: 45747
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.7.82.194"
port: 33631
}
bound_http_address {
host: "127.7.82.194"
port: 46397
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.7.82.195"
port: 37997
}
bound_http_address {
host: "127.7.82.195"
port: 40953
}
}
}
00:11:01.287 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
00:11:01.503 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
00:11:01.643 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task3KSwpE/test-tmp
00:11:01.644 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
00:11:01.644 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task3KSwpE/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
00:11:01.647 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
00:11:02.014 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
00:11:02.014 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
00:11:02.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:02.018230 8293 mini_oidc.cc:152] Starting JWKS server
00:11:02.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:02.021916 8293 webserver.cc:460] Webserver started at http://127.0.0.1:38077/ using document root <none> and password file <none>
00:11:02.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:02.022327 8293 mini_oidc.cc:189] Starting OIDC Discovery server
00:11:02.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:02.024019 8293 webserver.cc:460] Webserver started at http://127.0.0.1:36379/ using document root <none> and password file <none>
00:11:02.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
00:11:02.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/krb5kdc/principal' for realm 'KRBTEST.COM',
00:11:02.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
00:11:02.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 10 00:11:02 dist-test-slave-cbbz krb5kdc[8306](info): setting up network...
00:11:02.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
00:11:02.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 10 00:11:02 dist-test-slave-cbbz krb5kdc[8306](info): set up 2 sockets
00:11:02.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 10 00:11:02 dist-test-slave-cbbz krb5kdc[8306](info): commencing operation
00:11:02.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
00:11:02.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:02.233028 8284 heartbeater.cc:503] Master 127.7.82.254:41719 was elected leader, sending a full tablet report...
00:11:03.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:03.236027 8293 mini_kdc.cc:121] Time spent starting KDC: real 1.185s user 0.003s sys 0.020s
00:11:03.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
00:11:03.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
00:11:03.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
00:11:03.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
00:11:03.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
00:11:03.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
00:11:03.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:03.387825 8293 mini_kdc.cc:253] Time spent creating user principal test-user: real 0.115s user 0.001s sys 0.004s
00:11:03.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
00:11:03.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
00:11:03.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
00:11:03.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
00:11:03.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/krb5kdc/test-user.keytab.
00:11:03.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/krb5kdc/test-user.keytab.
00:11:03.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 10 00:11:03 dist-test-slave-cbbz krb5kdc[8306](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786320663, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
00:11:03.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
00:11:03.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-10T00:11:03Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
00:11:03.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-10T00:11:03Z Disabled control of system clock
00:11:03.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.8.25.126@KRBTEST.COM; defaulting to no policy
00:11:03.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
00:11:03.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.8.25.126@KRBTEST.COM" created.
00:11:03.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
00:11:03.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.8.25.126 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/krb5kdc/kudu_127.8.25.126.keytab.
00:11:03.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.8.25.126 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/krb5kdc/kudu_127.8.25.126.keytab.
00:11:03.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:03.548885 8293 external_mini_cluster.cc:1326] Running /tmp/dist-test-task3KSwpE/build/tsan/bin/kudu
00:11:03.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task3KSwpE/build/tsan/bin/kudu
00:11:03.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-0/wal
00:11:03.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-0/data
00:11:03.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:11:03.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:11:03.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:11:03.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:11:03.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:11:03.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:11:03.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-0/logs
00:11:03.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-0/data/info.pb
00:11:03.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:11:03.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:11:03.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:11:03.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:11:03.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:11:03.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:11:03.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:11:03.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:11:03.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
00:11:03.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:11:03.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:11:03.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:11:03.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.25.126:34221
00:11:03.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.25.126
00:11:03.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:11:03.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/krb5kdc/kudu_127.8.25.126.keytab
00:11:03.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.25.126
00:11:03.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
00:11:03.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
00:11:03.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
00:11:03.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.25.84:33871
00:11:03.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:11:03.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:11:03.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:11:03.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:11:03.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183
00:11:03.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:11:03.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38077/jwks.json
00:11:03.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
00:11:03.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:03.864830 8322 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
00:11:03.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:03.865363 8322 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:11:03.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:03.865922 8322 flags.cc:425] Enabled unsafe flag: --never_fsync=true
00:11:03.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:03.898211 8322 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
00:11:03.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:03.898432 8322 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
00:11:03.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:03.898687 8322 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
00:11:03.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:03.898921 8322 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
00:11:03.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:03.899164 8322 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
00:11:03.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:03.899353 8322 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
00:11:03.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:03.899490 8322 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38077/jwks.json
00:11:03.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:03.935549 8322 master_runner.cc:386] Master server non-default flags:
00:11:03.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:11:03.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.25.84:33871
00:11:03.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:11:03.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:11:03.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-0/data
00:11:03.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-0/wal
00:11:03.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
00:11:03.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:11:03.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183
00:11:03.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/krb5kdc/kudu_127.8.25.126.keytab
00:11:03.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.25.126
00:11:03.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:11:03.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:11:03.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:11:03.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.25.126:34221
00:11:03.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:11:03.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:11:03.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:11:03.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38077/jwks.json
00:11:03.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
00:11:03.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
00:11:03.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
00:11:03.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:11:03.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:11:03.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-0/data/info.pb
00:11:03.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.25.126
00:11:03.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:11:03.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:11:03.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:11:03.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:11:03.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:11:03.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:11:03.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-0/logs
00:11:03.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:11:03.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:11:03.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
00:11:03.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
00:11:03.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
00:11:03.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
00:11:03.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:11:03.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 23:15:58 UTC on ccycloud.quasar-ddmxrv.root.comops.site
00:11:03.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4576820
00:11:03.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
00:11:03.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:03.936797 8322 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:11:03.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:03.937876 8322 file_cache.cc:493] Constructed file cache file cache with capacity 419430
00:11:03.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:03.958406 8332 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:11:04.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:04.422524 8277 debug-util.cc:398] Leaking SignalData structure 0x7b08000af0a0 after lost signal to thread 8141
00:11:04.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:04.424098 8277 debug-util.cc:398] Leaking SignalData structure 0x7b08000b7640 after lost signal to thread 8281
00:11:05.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:03.963702 8329 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:11:05.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:03.959127 8330 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:11:05.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:05.320485 8331 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1358 milliseconds
00:11:05.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:05.320596 8322 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
00:11:05.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:05.322269 8322 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:11:05.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:05.324972 8322 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:11:05.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:05.326354 8322 hybrid_clock.cc:648] HybridClock initialized: now 1786320665326328 us; error 132 us; skew 500 ppm
00:11:05.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:05.327209 8322 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:11:05.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 10 00:11:05 dist-test-slave-cbbz krb5kdc[8306](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786320665, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.25.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
00:11:05.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:05.333912 8322 init.cc:377] Logged in from keytab as kudu/127.8.25.126@KRBTEST.COM (short username kudu)
00:11:05.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:05.339978 8322 webserver.cc:460] Webserver started at http://127.8.25.126:33123/ using document root <none> and password file <none>
00:11:05.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:05.341012 8322 fs_manager.cc:359] Metadata directory not provided
00:11:05.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:05.341252 8322 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:11:05.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:05.341827 8322 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
00:11:05.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:05.346875 8322 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-0/data/instance:
00:11:05.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2b03379f7f604a7d9bec7fac55417f0f"
00:11:05.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-10 00:11:05 on dist-test-slave-cbbz"
00:11:05.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:05.348109 8322 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-0/wal/instance:
00:11:05.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2b03379f7f604a7d9bec7fac55417f0f"
00:11:05.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-10 00:11:05 on dist-test-slave-cbbz"
00:11:05.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:05.356837 8322 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.007s sys 0.001s
00:11:05.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:05.363615 8344 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
00:11:05.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:05.365025 8322 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.002s sys 0.001s
00:11:05.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:05.365423 8322 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-0/data,/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-0/wal
00:11:05.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2b03379f7f604a7d9bec7fac55417f0f"
00:11:05.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-10 00:11:05 on dist-test-slave-cbbz"
00:11:05.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:05.365772 8322 fs_report.cc:383] FS layout report
00:11:05.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:11:05.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-0/wal
00:11:05.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-0/wal
00:11:05.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-0/data/data
00:11:05.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:11:05.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:11:05.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:11:05.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:11:05.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:11:05.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:11:05.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:11:05.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:11:05.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:11:05.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:11:05.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:11:05.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:05.412663 8322 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
00:11:05.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:05.419162 8322 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:11:05.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:05.419713 8322 kserver.cc:163] Server-wide thread pool size limit: 3276
00:11:05.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:05.516585 8322 rpc_server.cc:307] RPC server started. Bound to: 127.8.25.126:34221
00:11:05.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:05.516686 8397 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.8.25.126:34221 every 8 connection(s)
00:11:05.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:05.519375 8322 server_base.cc:1166] Dumped server information to /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-0/data/info.pb
00:11:05.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:05.525787 8398 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
00:11:05.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:05.527613 8293 external_mini_cluster.cc:1388] Started /tmp/dist-test-task3KSwpE/build/tsan/bin/kudu as pid 8322
00:11:05.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:05.528175 8293 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-0/wal/instance
00:11:05.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:05.542008 8398 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.25.126" port: 34221 } has no permanent_uuid. Determining permanent_uuid...
00:11:05.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.8.25.125@KRBTEST.COM; defaulting to no policy
00:11:05.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
00:11:05.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.8.25.125@KRBTEST.COM" created.
00:11:05.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
00:11:05.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.8.25.125 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/krb5kdc/kudu_127.8.25.125.keytab.
00:11:05.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.8.25.125 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/krb5kdc/kudu_127.8.25.125.keytab.
00:11:05.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:05.595403 8293 external_mini_cluster.cc:1326] Running /tmp/dist-test-task3KSwpE/build/tsan/bin/kudu
00:11:05.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task3KSwpE/build/tsan/bin/kudu
00:11:05.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-1/wal
00:11:05.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-1/data
00:11:05.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:11:05.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:11:05.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:11:05.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:11:05.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:11:05.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:11:05.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-1/logs
00:11:05.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-1/data/info.pb
00:11:05.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:11:05.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:11:05.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:11:05.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:11:05.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:11:05.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:11:05.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:11:05.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:11:05.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
00:11:05.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:11:05.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:11:05.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:11:05.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.25.125:44265
00:11:05.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.25.125
00:11:05.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:11:05.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/krb5kdc/kudu_127.8.25.125.keytab
00:11:05.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.25.125
00:11:05.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
00:11:05.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
00:11:05.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
00:11:05.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.25.84:33871
00:11:05.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:11:05.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:11:05.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:11:05.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:11:05.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183
00:11:05.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:11:05.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38077/jwks.json
00:11:05.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
00:11:05.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 10 00:11:05 dist-test-slave-cbbz krb5kdc[8306](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786320665, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.25.126@KRBTEST.COM for kudu/127.8.25.126@KRBTEST.COM
00:11:05.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:05.614784 8398 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.25.125" port: 44265 } has no permanent_uuid. Determining permanent_uuid...
00:11:05.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:05.617709 8347 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.25.125:44265: connect: Connection refused (error 111)
00:11:05.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:05.623386 8398 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.25.125:44265: Network error: Client connection negotiation failed: client connection to 127.8.25.125:44265: connect: Connection refused (error 111)
00:11:05.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:05.672986 8398 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.25.125" port: 44265 } attempt: 1
00:11:05.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:05.677666 8398 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.25.125:44265: Network error: Client connection negotiation failed: client connection to 127.8.25.125:44265: connect: Connection refused (error 111)
00:11:05.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:05.746153 8398 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.25.125" port: 44265 } attempt: 2
00:11:05.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:05.750895 8398 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.25.125:44265: Network error: Client connection negotiation failed: client connection to 127.8.25.125:44265: connect: Connection refused (error 111)
00:11:05.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:05.842350 8398 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.25.125" port: 44265 } attempt: 3
00:11:05.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:05.846464 8398 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.25.125:44265: Network error: Client connection negotiation failed: client connection to 127.8.25.125:44265: connect: Connection refused (error 111)
00:11:05.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:05.989832 8398 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.25.125" port: 44265 } attempt: 4
00:11:05.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:05.993957 8398 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.25.125:44265: Network error: Client connection negotiation failed: client connection to 127.8.25.125:44265: connect: Connection refused (error 111)
00:11:06.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:06.084898 8405 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
00:11:06.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:06.085841 8405 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:11:06.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:06.087106 8405 flags.cc:425] Enabled unsafe flag: --never_fsync=true
00:11:06.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:06.126399 8405 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
00:11:06.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:06.126670 8405 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
00:11:06.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:06.126974 8405 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
00:11:06.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:06.127192 8405 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
00:11:06.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:06.127398 8405 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
00:11:06.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:06.127612 8405 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
00:11:06.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:06.127763 8405 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38077/jwks.json
00:11:06.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:06.169116 8405 master_runner.cc:386] Master server non-default flags:
00:11:06.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:11:06.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.25.84:33871
00:11:06.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:11:06.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:11:06.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-1/data
00:11:06.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-1/wal
00:11:06.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
00:11:06.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:11:06.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183
00:11:06.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/krb5kdc/kudu_127.8.25.125.keytab
00:11:06.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.25.125
00:11:06.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:11:06.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:11:06.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:11:06.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.25.125:44265
00:11:06.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:11:06.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:11:06.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:11:06.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38077/jwks.json
00:11:06.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
00:11:06.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
00:11:06.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
00:11:06.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:11:06.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:11:06.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-1/data/info.pb
00:11:06.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.25.125
00:11:06.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:11:06.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:11:06.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:11:06.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:11:06.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:11:06.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:11:06.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-1/logs
00:11:06.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:11:06.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:11:06.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
00:11:06.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
00:11:06.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
00:11:06.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
00:11:06.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:11:06.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 23:15:58 UTC on ccycloud.quasar-ddmxrv.root.comops.site
00:11:06.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4576820
00:11:06.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
00:11:06.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:06.170332 8405 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:11:06.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:06.171422 8405 file_cache.cc:493] Constructed file cache file cache with capacity 419430
00:11:06.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:06.194916 8414 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:11:06.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:06.293370 8398 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.25.125" port: 44265 } attempt: 5
00:11:06.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:06.299053 8398 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.25.125:44265: Network error: Client connection negotiation failed: client connection to 127.8.25.125:44265: connect: Connection refused (error 111)
00:11:06.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:06.846565 8398 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.25.125" port: 44265 } attempt: 6
00:11:06.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:06.856920 8398 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.25.125:44265: Network error: Client connection negotiation failed: client connection to 127.8.25.125:44265: connect: Connection refused (error 111)
00:11:07.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:06.196553 8417 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:11:07.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:06.194978 8415 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:11:07.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:07.536377 8405 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
00:11:07.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:07.536808 8416 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
00:11:07.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:07.541016 8405 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:11:07.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:07.543807 8405 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:11:07.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:07.545281 8405 hybrid_clock.cc:648] HybridClock initialized: now 1786320667545252 us; error 64 us; skew 500 ppm
00:11:07.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:07.546162 8405 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:11:07.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 10 00:11:07 dist-test-slave-cbbz krb5kdc[8306](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786320667, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.25.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
00:11:07.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:07.556546 8405 init.cc:377] Logged in from keytab as kudu/127.8.25.125@KRBTEST.COM (short username kudu)
00:11:07.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:07.562455 8405 webserver.cc:460] Webserver started at http://127.8.25.125:42261/ using document root <none> and password file <none>
00:11:07.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:07.563378 8405 fs_manager.cc:359] Metadata directory not provided
00:11:07.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:07.563623 8405 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:11:07.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:07.564057 8405 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
00:11:07.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:07.569087 8405 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-1/data/instance:
00:11:07.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "88c642733e5b44adaefc882d98c8d199"
00:11:07.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-10 00:11:07 on dist-test-slave-cbbz"
00:11:07.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:07.570233 8405 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-1/wal/instance:
00:11:07.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "88c642733e5b44adaefc882d98c8d199"
00:11:07.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-10 00:11:07 on dist-test-slave-cbbz"
00:11:07.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:07.577804 8405 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.004s sys 0.004s
00:11:07.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:07.583585 8428 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
00:11:07.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:07.584795 8405 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
00:11:07.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:07.585093 8405 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-1/data,/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-1/wal
00:11:07.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "88c642733e5b44adaefc882d98c8d199"
00:11:07.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-10 00:11:07 on dist-test-slave-cbbz"
00:11:07.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:07.585397 8405 fs_report.cc:383] FS layout report
00:11:07.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:11:07.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-1/wal
00:11:07.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-1/wal
00:11:07.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-1/data/data
00:11:07.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:11:07.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:11:07.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:11:07.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:11:07.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:11:07.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:11:07.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:11:07.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:11:07.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:11:07.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:11:07.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:11:07.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:07.639441 8405 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
00:11:07.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:07.645642 8405 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:11:07.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:07.646085 8405 kserver.cc:163] Server-wide thread pool size limit: 3276
00:11:07.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:07.742237 8405 rpc_server.cc:307] RPC server started. Bound to: 127.8.25.125:44265
00:11:07.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:07.742363 8484 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.8.25.125:44265 every 8 connection(s)
00:11:07.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:07.745296 8405 server_base.cc:1166] Dumped server information to /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-1/data/info.pb
00:11:07.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:07.751631 8486 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
00:11:07.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:07.754469 8293 external_mini_cluster.cc:1388] Started /tmp/dist-test-task3KSwpE/build/tsan/bin/kudu as pid 8405
00:11:07.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:07.754922 8293 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-1/wal/instance
00:11:07.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:07.765357 8486 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.25.126" port: 34221 } has no permanent_uuid. Determining permanent_uuid...
00:11:07.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.8.25.124@KRBTEST.COM; defaulting to no policy
00:11:07.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
00:11:07.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.8.25.124@KRBTEST.COM" created.
00:11:07.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 10 00:11:07 dist-test-slave-cbbz krb5kdc[8306](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786320667, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.25.125@KRBTEST.COM for kudu/127.8.25.126@KRBTEST.COM
00:11:07.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
00:11:07.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.8.25.124 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/krb5kdc/kudu_127.8.25.124.keytab.
00:11:07.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.8.25.124 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/krb5kdc/kudu_127.8.25.124.keytab.
00:11:07.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:07.809469 8293 external_mini_cluster.cc:1326] Running /tmp/dist-test-task3KSwpE/build/tsan/bin/kudu
00:11:07.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task3KSwpE/build/tsan/bin/kudu
00:11:07.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-2/wal
00:11:07.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-2/data
00:11:07.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:11:07.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:11:07.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:11:07.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:11:07.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:11:07.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:11:07.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-2/logs
00:11:07.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-2/data/info.pb
00:11:07.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:11:07.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:11:07.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:11:07.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:11:07.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:11:07.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:11:07.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:11:07.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:11:07.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
00:11:07.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:11:07.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:11:07.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:11:07.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.25.124:36183
00:11:07.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.25.124
00:11:07.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:11:07.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/krb5kdc/kudu_127.8.25.124.keytab
00:11:07.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.25.124
00:11:07.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
00:11:07.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
00:11:07.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
00:11:07.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.25.84:33871
00:11:07.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:11:07.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:11:07.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:11:07.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:11:07.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183
00:11:07.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:11:07.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38077/jwks.json
00:11:07.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
00:11:07.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:07.809868 8486 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.25.125" port: 44265 } has no permanent_uuid. Determining permanent_uuid...
00:11:07.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 10 00:11:07 dist-test-slave-cbbz krb5kdc[8306](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786320667, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.25.125@KRBTEST.COM for kudu/127.8.25.125@KRBTEST.COM
00:11:07.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:07.839596 8486 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.25.124" port: 36183 } has no permanent_uuid. Determining permanent_uuid...
00:11:07.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:07.842130 8429 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.25.124:36183: connect: Connection refused (error 111)
00:11:07.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:07.846522 8486 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.25.124:36183: Network error: Client connection negotiation failed: client connection to 127.8.25.124:36183: connect: Connection refused (error 111)
00:11:07.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:07.896004 8486 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.25.124" port: 36183 } attempt: 1
00:11:07.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:07.900807 8486 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.25.124:36183: Network error: Client connection negotiation failed: client connection to 127.8.25.124:36183: connect: Connection refused (error 111)
00:11:07.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:07.917532 8398 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.25.125" port: 44265 } attempt: 7
00:11:07.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 10 00:11:07 dist-test-slave-cbbz krb5kdc[8306](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786320665, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.25.126@KRBTEST.COM for kudu/127.8.25.125@KRBTEST.COM
00:11:07.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:07.943212 8398 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.25.124" port: 36183 } has no permanent_uuid. Determining permanent_uuid...
00:11:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:07.949440 8398 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.25.124:36183: Network error: Client connection negotiation failed: client connection to 127.8.25.124:36183: connect: Connection refused (error 111)
00:11:07.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:07.969280 8486 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.25.124" port: 36183 } attempt: 2
00:11:07.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:07.973709 8486 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.25.124:36183: Network error: Client connection negotiation failed: client connection to 127.8.25.124:36183: connect: Connection refused (error 111)
00:11:08.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:08.007867 8398 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.25.124" port: 36183 } attempt: 1
00:11:08.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:08.012589 8398 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.25.124:36183: Network error: Client connection negotiation failed: client connection to 127.8.25.124:36183: connect: Connection refused (error 111)
00:11:08.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:08.065171 8486 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.25.124" port: 36183 } attempt: 3
00:11:08.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:08.069280 8486 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.25.124:36183: Network error: Client connection negotiation failed: client connection to 127.8.25.124:36183: connect: Connection refused (error 111)
00:11:08.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:08.093999 8398 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.25.124" port: 36183 } attempt: 2
00:11:08.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:08.098444 8398 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.25.124:36183: Network error: Client connection negotiation failed: client connection to 127.8.25.124:36183: connect: Connection refused (error 111)
00:11:08.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:08.175659 8493 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
00:11:08.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:08.176368 8493 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:11:08.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:08.176993 8493 flags.cc:425] Enabled unsafe flag: --never_fsync=true
00:11:08.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:08.183900 8398 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.25.124" port: 36183 } attempt: 3
00:11:08.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:08.188058 8398 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.25.124:36183: Network error: Client connection negotiation failed: client connection to 127.8.25.124:36183: connect: Connection refused (error 111)
00:11:08.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:08.211575 8493 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
00:11:08.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:08.211807 8493 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
00:11:08.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:08.212071 8493 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
00:11:08.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:08.212357 8493 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
00:11:08.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:08.212553 8493 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
00:11:08.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:08.212742 8493 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
00:11:08.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:08.212880 8493 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38077/jwks.json
00:11:08.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:08.212698 8486 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.25.124" port: 36183 } attempt: 4
00:11:08.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:08.216566 8486 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.25.124:36183: Network error: Client connection negotiation failed: client connection to 127.8.25.124:36183: connect: Connection refused (error 111)
00:11:08.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:08.258180 8493 master_runner.cc:386] Master server non-default flags:
00:11:08.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:11:08.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.25.84:33871
00:11:08.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:11:08.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:11:08.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-2/data
00:11:08.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-2/wal
00:11:08.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
00:11:08.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:11:08.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183
00:11:08.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/krb5kdc/kudu_127.8.25.124.keytab
00:11:08.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.25.124
00:11:08.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:11:08.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:11:08.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:11:08.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.25.124:36183
00:11:08.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:11:08.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:11:08.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:11:08.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38077/jwks.json
00:11:08.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
00:11:08.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
00:11:08.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
00:11:08.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:11:08.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:11:08.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-2/data/info.pb
00:11:08.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.25.124
00:11:08.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:11:08.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:11:08.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:11:08.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:11:08.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:11:08.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:11:08.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-2/logs
00:11:08.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:11:08.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:11:08.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
00:11:08.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
00:11:08.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
00:11:08.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
00:11:08.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:11:08.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 23:15:58 UTC on ccycloud.quasar-ddmxrv.root.comops.site
00:11:08.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4576820
00:11:08.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
00:11:08.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:08.259357 8493 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:11:08.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:08.260433 8493 file_cache.cc:493] Constructed file cache file cache with capacity 419430
00:11:08.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:08.280203 8505 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:11:08.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:08.328490 8398 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.25.124" port: 36183 } attempt: 4
00:11:08.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:08.332799 8398 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.25.124:36183: Network error: Client connection negotiation failed: client connection to 127.8.25.124:36183: connect: Connection refused (error 111)
00:11:08.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:08.515938 8486 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.25.124" port: 36183 } attempt: 5
00:11:08.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:08.519637 8486 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.25.124:36183: Network error: Client connection negotiation failed: client connection to 127.8.25.124:36183: connect: Connection refused (error 111)
00:11:08.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:08.616236 8398 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.25.124" port: 36183 } attempt: 5
00:11:08.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:08.619980 8398 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.25.124:36183: Network error: Client connection negotiation failed: client connection to 127.8.25.124:36183: connect: Connection refused (error 111)
00:11:09.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:09.067055 8486 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.25.124" port: 36183 } attempt: 6
00:11:09.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:09.074008 8486 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.25.124:36183: Network error: Client connection negotiation failed: client connection to 127.8.25.124:36183: connect: Connection refused (error 111)
00:11:09.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:09.172665 8398 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.25.124" port: 36183 } attempt: 6
00:11:09.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:09.180789 8398 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.25.124:36183: Network error: Client connection negotiation failed: client connection to 127.8.25.124:36183: connect: Connection refused (error 111)
00:11:09.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:08.280859 8508 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:11:09.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:08.281008 8504 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:11:09.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:08.282404 8493 server_base.cc:1034] running on GCE node
00:11:09.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:09.548979 8493 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:11:09.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:09.552264 8493 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:11:09.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:09.553635 8493 hybrid_clock.cc:648] HybridClock initialized: now 1786320669553594 us; error 60 us; skew 500 ppm
00:11:09.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:09.554664 8493 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:11:09.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 10 00:11:09 dist-test-slave-cbbz krb5kdc[8306](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786320669, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.25.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
00:11:09.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:09.560755 8493 init.cc:377] Logged in from keytab as kudu/127.8.25.124@KRBTEST.COM (short username kudu)
00:11:09.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:09.566677 8493 webserver.cc:460] Webserver started at http://127.8.25.124:35221/ using document root <none> and password file <none>
00:11:09.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:09.567549 8493 fs_manager.cc:359] Metadata directory not provided
00:11:09.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:09.567739 8493 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:11:09.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:09.568185 8493 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
00:11:09.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:09.573173 8493 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-2/data/instance:
00:11:09.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "69741f4e09d34174a4da10c0d0ccac0f"
00:11:09.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-10 00:11:09 on dist-test-slave-cbbz"
00:11:09.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:09.574329 8493 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-2/wal/instance:
00:11:09.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "69741f4e09d34174a4da10c0d0ccac0f"
00:11:09.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-10 00:11:09 on dist-test-slave-cbbz"
00:11:09.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:09.583149 8493 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.001s sys 0.008s
00:11:09.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:09.589253 8522 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
00:11:09.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:09.590479 8493 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
00:11:09.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:09.590763 8493 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-2/data,/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-2/wal
00:11:09.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "69741f4e09d34174a4da10c0d0ccac0f"
00:11:09.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-10 00:11:09 on dist-test-slave-cbbz"
00:11:09.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:09.591061 8493 fs_report.cc:383] FS layout report
00:11:09.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:11:09.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-2/wal
00:11:09.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-2/wal
00:11:09.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-2/data/data
00:11:09.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:11:09.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:11:09.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:11:09.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:11:09.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:11:09.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:11:09.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:11:09.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:11:09.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:11:09.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:11:09.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:11:09.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:09.636596 8493 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
00:11:09.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:09.643926 8493 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:11:09.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:09.644472 8493 kserver.cc:163] Server-wide thread pool size limit: 3276
00:11:09.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:09.727934 8493 rpc_server.cc:307] RPC server started. Bound to: 127.8.25.124:36183
00:11:09.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:09.727985 8582 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.8.25.124:36183 every 8 connection(s)
00:11:09.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:09.731709 8493 server_base.cc:1166] Dumped server information to /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-2/data/info.pb
00:11:09.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:09.734972 8293 external_mini_cluster.cc:1388] Started /tmp/dist-test-task3KSwpE/build/tsan/bin/kudu as pid 8493
00:11:09.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:09.735462 8293 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-2/wal/instance
00:11:09.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:09.745311 8583 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
00:11:09.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:09.771473 8583 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.25.126" port: 34221 } has no permanent_uuid. Determining permanent_uuid...
00:11:09.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 10 00:11:09 dist-test-slave-cbbz krb5kdc[8306](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786320663, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.8.25.124@KRBTEST.COM
00:11:09.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 10 00:11:09 dist-test-slave-cbbz krb5kdc[8306](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786320669, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.25.124@KRBTEST.COM for kudu/127.8.25.126@KRBTEST.COM
00:11:09.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:09.811890 8583 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.25.125" port: 44265 } has no permanent_uuid. Determining permanent_uuid...
00:11:09.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 10 00:11:09 dist-test-slave-cbbz krb5kdc[8306](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786320663, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.8.25.125@KRBTEST.COM
00:11:09.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 10 00:11:09 dist-test-slave-cbbz krb5kdc[8306](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786320669, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.25.124@KRBTEST.COM for kudu/127.8.25.125@KRBTEST.COM
00:11:09.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:09.840766 8583 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.25.124" port: 36183 } has no permanent_uuid. Determining permanent_uuid...
00:11:09.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 10 00:11:09 dist-test-slave-cbbz krb5kdc[8306](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786320669, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.25.124@KRBTEST.COM for kudu/127.8.25.124@KRBTEST.COM
00:11:09.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 10 00:11:09 dist-test-slave-cbbz krb5kdc[8306](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786320663, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.8.25.126@KRBTEST.COM
00:11:09.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:09.873451 8583 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f: Bootstrap starting.
00:11:09.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:09.874893 8293 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
00:11:09.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:09.880429 8583 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f: Neither blocks nor log segments found. Creating new log.
00:11:09.882 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
00:11:09.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:09.882459 8583 log.cc:824] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f: Log is configured to *not* fsync() on all Append() calls
00:11:09.884 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
00:11:09.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:09.888904 8583 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f: No bootstrap required, opened a new log
00:11:09.895 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
00:11:09.896 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
00:11:09.900 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.8.25.126"
port: 34221
}
bound_http_address {
host: "127.8.25.126"
port: 33123
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.8.25.125"
port: 44265
}
bound_http_address {
host: "127.8.25.125"
port: 42261
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.8.25.124"
port: 36183
}
bound_http_address {
host: "127.8.25.124"
port: 35221
}
}
}
00:11:09.901 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
00:11:09.901 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
00:11:09.901 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
00:11:09.903 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
00:11:09.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:09.915539 8583 raft_consensus.cc:348] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2b03379f7f604a7d9bec7fac55417f0f" member_type: VOTER last_known_addr { host: "127.8.25.126" port: 34221 } } peers { permanent_uuid: "88c642733e5b44adaefc882d98c8d199" member_type: VOTER last_known_addr { host: "127.8.25.125" port: 44265 } } peers { permanent_uuid: "69741f4e09d34174a4da10c0d0ccac0f" member_type: VOTER last_known_addr { host: "127.8.25.124" port: 36183 } }
00:11:09.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:09.916605 8583 raft_consensus.cc:374] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
00:11:09.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:09.916929 8583 raft_consensus.cc:729] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 69741f4e09d34174a4da10c0d0ccac0f, State: Initialized, Role: FOLLOWER
00:11:09.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:09.919454 8583 consensus_queue.cc:260] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2b03379f7f604a7d9bec7fac55417f0f" member_type: VOTER last_known_addr { host: "127.8.25.126" port: 34221 } } peers { permanent_uuid: "88c642733e5b44adaefc882d98c8d199" member_type: VOTER last_known_addr { host: "127.8.25.125" port: 44265 } } peers { permanent_uuid: "69741f4e09d34174a4da10c0d0ccac0f" member_type: VOTER last_known_addr { host: "127.8.25.124" port: 36183 } }
00:11:09.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:09.923343 8595 sys_catalog.cc:455] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2b03379f7f604a7d9bec7fac55417f0f" member_type: VOTER last_known_addr { host: "127.8.25.126" port: 34221 } } peers { permanent_uuid: "88c642733e5b44adaefc882d98c8d199" member_type: VOTER last_known_addr { host: "127.8.25.125" port: 44265 } } peers { permanent_uuid: "69741f4e09d34174a4da10c0d0ccac0f" member_type: VOTER last_known_addr { host: "127.8.25.124" port: 36183 } } }
00:11:09.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:09.924548 8595 sys_catalog.cc:458] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f [sys.catalog]: This master's current role is: FOLLOWER
00:11:09.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:09.925362 8583 sys_catalog.cc:564] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f [sys.catalog]: configured and running, proceeding with master startup.
00:11:09.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 10 00:11:09 dist-test-slave-cbbz krb5kdc[8306](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786320669, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
00:11:09.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
00:11:09.959 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
00:11:09.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:09.964223 8608 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:11:09.966 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/krb5kdc/krb5cc
00:11:09.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:09.970211 8608 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:11:09.974 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
00:11:09.975 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/krb5kdc/krb5cc
00:11:09.977 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
00:11:09.981 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
00:11:09.983 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
00:11:09.984 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
00:11:09.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:09.958137 8493 master_runner.cc:186] Error getting master registration for 127.8.25.126:34221: OK, instance_id { permanent_uuid: "2b03379f7f604a7d9bec7fac55417f0f" instance_seqno: 1786320665490606 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
00:11:09.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:09.994274 8493 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
00:11:10.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 10 00:11:10 dist-test-slave-cbbz krb5kdc[8306](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786320670, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
00:11:10.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
00:11:10.025 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
00:11:10.026 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/krb5kdc/krb5cc
00:11:10.027 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
00:11:10.028 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.8.25.126:34221
00:11:10.029 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
00:11:10.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:10.030246 8293 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task3KSwpE/build/tsan/bin/kudu with pid 8322
00:11:10.057 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
00:11:10.058 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.8.25.125:44265
00:11:10.058 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
00:11:10.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:10.058815 8293 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task3KSwpE/build/tsan/bin/kudu with pid 8405
00:11:10.089 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
00:11:10.089 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.8.25.124:36183
00:11:10.089 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
00:11:10.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:10.089974 8293 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task3KSwpE/build/tsan/bin/kudu with pid 8493
00:11:10.110 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
00:11:10.110 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.8.25.126:34221
00:11:10.112 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
00:11:10.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:10.113776 8293 external_mini_cluster.cc:1326] Running /tmp/dist-test-task3KSwpE/build/tsan/bin/kudu
00:11:10.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task3KSwpE/build/tsan/bin/kudu
00:11:10.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-0/wal
00:11:10.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-0/data
00:11:10.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:11:10.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:11:10.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:11:10.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:11:10.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:11:10.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:11:10.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-0/logs
00:11:10.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-0/data/info.pb
00:11:10.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:11:10.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:11:10.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:11:10.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:11:10.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:11:10.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:11:10.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:11:10.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:11:10.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
00:11:10.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:11:10.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:11:10.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:11:10.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.25.126:34221
00:11:10.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.25.126
00:11:10.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33123
00:11:10.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/krb5kdc/kudu_127.8.25.126.keytab
00:11:10.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.25.126
00:11:10.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
00:11:10.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
00:11:10.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
00:11:10.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.25.84:33871
00:11:10.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:11:10.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:11:10.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:11:10.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:11:10.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183
00:11:10.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:11:10.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38077/jwks.json
00:11:10.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
00:11:10.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:10.439106 8613 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
00:11:10.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:10.439795 8613 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:11:10.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:10.440872 8613 flags.cc:425] Enabled unsafe flag: --never_fsync=true
00:11:10.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:10.473601 8613 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
00:11:10.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:10.473910 8613 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
00:11:10.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:10.474445 8613 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
00:11:10.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:10.474787 8613 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
00:11:10.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:10.475149 8613 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
00:11:10.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:10.475461 8613 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
00:11:10.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:10.475761 8613 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38077/jwks.json
00:11:10.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:10.512650 8613 master_runner.cc:386] Master server non-default flags:
00:11:10.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:11:10.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.25.84:33871
00:11:10.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:11:10.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:11:10.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-0/data
00:11:10.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-0/wal
00:11:10.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
00:11:10.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:11:10.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183
00:11:10.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/krb5kdc/kudu_127.8.25.126.keytab
00:11:10.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.25.126
00:11:10.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:11:10.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:11:10.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:11:10.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.25.126:34221
00:11:10.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:11:10.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:11:10.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:11:10.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38077/jwks.json
00:11:10.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
00:11:10.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
00:11:10.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
00:11:10.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:11:10.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:11:10.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-0/data/info.pb
00:11:10.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.25.126
00:11:10.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33123
00:11:10.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:11:10.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:11:10.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:11:10.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:11:10.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:11:10.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-0/logs
00:11:10.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:11:10.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:11:10.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
00:11:10.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
00:11:10.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
00:11:10.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
00:11:10.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:11:10.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 23:15:58 UTC on ccycloud.quasar-ddmxrv.root.comops.site
00:11:10.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4576820
00:11:10.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
00:11:10.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:10.514129 8613 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:11:10.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:10.515686 8613 file_cache.cc:493] Constructed file cache file cache with capacity 419430
00:11:10.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:10.548060 8621 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:11:11.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:10.549015 8623 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:11:11.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:10.548239 8620 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:11:11.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:11.848198 8622 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1298 milliseconds
00:11:11.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:11.848327 8613 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
00:11:11.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:11.849771 8613 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:11:11.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:11.853005 8613 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:11:11.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:11.854483 8613 hybrid_clock.cc:648] HybridClock initialized: now 1786320671854382 us; error 114 us; skew 500 ppm
00:11:11.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:11.855377 8613 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:11:11.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 10 00:11:11 dist-test-slave-cbbz krb5kdc[8306](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786320671, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.25.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
00:11:11.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:11.861439 8613 init.cc:377] Logged in from keytab as kudu/127.8.25.126@KRBTEST.COM (short username kudu)
00:11:11.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:11.866711 8613 webserver.cc:460] Webserver started at http://127.8.25.126:33123/ using document root <none> and password file <none>
00:11:11.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:11.867645 8613 fs_manager.cc:359] Metadata directory not provided
00:11:11.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:11.867846 8613 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:11:11.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:11.875695 8613 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.006s sys 0.001s
00:11:11.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:11.880236 8633 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
00:11:11.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:11.881343 8613 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
00:11:11.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:11.881686 8613 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-0/data,/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-0/wal
00:11:11.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2b03379f7f604a7d9bec7fac55417f0f"
00:11:11.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-10 00:11:05 on dist-test-slave-cbbz"
00:11:11.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:11.884320 8613 fs_report.cc:383] FS layout report
00:11:11.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:11:11.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-0/wal
00:11:11.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-0/wal
00:11:11.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-0/data/data
00:11:11.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:11:11.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:11:11.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:11:11.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:11:11.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:11:11.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:11:11.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:11:11.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:11:11.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:11:11.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:11:11.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:11:11.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:11.926162 8613 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
00:11:11.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:11.932789 8613 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:11:11.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:11.933386 8613 kserver.cc:163] Server-wide thread pool size limit: 3276
00:11:12.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:12.012820 8613 rpc_server.cc:307] RPC server started. Bound to: 127.8.25.126:34221
00:11:12.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:12.012957 8691 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.8.25.126:34221 every 8 connection(s)
00:11:12.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:12.015733 8613 server_base.cc:1166] Dumped server information to /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-0/data/info.pb
00:11:12.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:12.021493 8293 external_mini_cluster.cc:1388] Started /tmp/dist-test-task3KSwpE/build/tsan/bin/kudu as pid 8613
00:11:12.022 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
00:11:12.022 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.8.25.125:44265
00:11:12.022 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
00:11:12.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:12.023980 8293 external_mini_cluster.cc:1326] Running /tmp/dist-test-task3KSwpE/build/tsan/bin/kudu
00:11:12.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task3KSwpE/build/tsan/bin/kudu
00:11:12.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-1/wal
00:11:12.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-1/data
00:11:12.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:11:12.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:11:12.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:11:12.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:11:12.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:11:12.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:11:12.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-1/logs
00:11:12.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-1/data/info.pb
00:11:12.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:11:12.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:11:12.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:11:12.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:11:12.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:11:12.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:11:12.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:11:12.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:11:12.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
00:11:12.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:11:12.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:11:12.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:11:12.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.25.125:44265
00:11:12.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.25.125
00:11:12.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42261
00:11:12.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/krb5kdc/kudu_127.8.25.125.keytab
00:11:12.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.25.125
00:11:12.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
00:11:12.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
00:11:12.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
00:11:12.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.25.84:33871
00:11:12.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:11:12.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:11:12.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:11:12.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:11:12.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183
00:11:12.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:11:12.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38077/jwks.json
00:11:12.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
00:11:12.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:12.026106 8692 sys_catalog.cc:263] Verifying existing consensus state
00:11:12.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260810 00:11:12.026767 8692 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
00:11:12.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:12.027132 8613 master.cc:537] Master@127.8.25.126:34221 shutting down...
00:11:12.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:12.109396 8613 master.cc:552] Master@127.8.25.126:34221 shutdown complete.
00:11:12.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
00:11:12.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:12.359840 8694 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
00:11:12.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:12.360569 8694 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:11:12.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:12.361318 8694 flags.cc:425] Enabled unsafe flag: --never_fsync=true
00:11:12.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:12.393644 8694 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
00:11:12.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:12.393971 8694 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
00:11:12.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:12.394428 8694 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
00:11:12.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:12.394779 8694 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
00:11:12.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:12.395107 8694 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
00:11:12.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:12.395433 8694 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
00:11:12.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:12.395674 8694 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38077/jwks.json
00:11:12.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:12.437661 8694 master_runner.cc:386] Master server non-default flags:
00:11:12.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:11:12.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.25.84:33871
00:11:12.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:11:12.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:11:12.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-1/data
00:11:12.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-1/wal
00:11:12.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
00:11:12.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:11:12.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183
00:11:12.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/krb5kdc/kudu_127.8.25.125.keytab
00:11:12.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.25.125
00:11:12.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:11:12.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:11:12.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:11:12.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.25.125:44265
00:11:12.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:11:12.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:11:12.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:11:12.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38077/jwks.json
00:11:12.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
00:11:12.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
00:11:12.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
00:11:12.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:11:12.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:11:12.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-1/data/info.pb
00:11:12.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.25.125
00:11:12.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42261
00:11:12.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:11:12.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:11:12.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:11:12.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:11:12.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:11:12.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-1/logs
00:11:12.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:11:12.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:11:12.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
00:11:12.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
00:11:12.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
00:11:12.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
00:11:12.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:11:12.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 23:15:58 UTC on ccycloud.quasar-ddmxrv.root.comops.site
00:11:12.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4576820
00:11:12.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
00:11:12.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:12.439248 8694 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:11:12.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:12.440694 8694 file_cache.cc:493] Constructed file cache file cache with capacity 419430
00:11:12.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:12.458016 8702 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:11:13.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:12.458127 8703 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:11:13.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:12.458042 8706 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:11:13.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:13.742558 8705 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
00:11:13.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:13.743181 8694 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
00:11:13.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:13.747092 8694 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:11:13.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:13.750058 8694 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:11:13.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:13.751484 8694 hybrid_clock.cc:648] HybridClock initialized: now 1786320673751402 us; error 85 us; skew 500 ppm
00:11:13.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:13.752455 8694 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:11:13.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 10 00:11:13 dist-test-slave-cbbz krb5kdc[8306](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786320673, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.25.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
00:11:13.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:13.758627 8694 init.cc:377] Logged in from keytab as kudu/127.8.25.125@KRBTEST.COM (short username kudu)
00:11:13.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:13.764245 8694 webserver.cc:460] Webserver started at http://127.8.25.125:42261/ using document root <none> and password file <none>
00:11:13.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:13.765187 8694 fs_manager.cc:359] Metadata directory not provided
00:11:13.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:13.765450 8694 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:11:13.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:13.773528 8694 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.003s sys 0.001s
00:11:13.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:13.778060 8715 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
00:11:13.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:13.779198 8694 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
00:11:13.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:13.779482 8694 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-1/data,/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-1/wal
00:11:13.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "88c642733e5b44adaefc882d98c8d199"
00:11:13.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-10 00:11:07 on dist-test-slave-cbbz"
00:11:13.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:13.782109 8694 fs_report.cc:383] FS layout report
00:11:13.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:11:13.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-1/wal
00:11:13.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-1/wal
00:11:13.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-1/data/data
00:11:13.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:11:13.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:11:13.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:11:13.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:11:13.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:11:13.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:11:13.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:11:13.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:11:13.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:11:13.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:11:13.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:11:13.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:13.816697 8694 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
00:11:13.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:13.822471 8694 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:11:13.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:13.822906 8694 kserver.cc:163] Server-wide thread pool size limit: 3276
00:11:13.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:13.895071 8694 rpc_server.cc:307] RPC server started. Bound to: 127.8.25.125:44265
00:11:13.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:13.895151 8771 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.8.25.125:44265 every 8 connection(s)
00:11:13.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:13.897907 8694 server_base.cc:1166] Dumped server information to /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-1/data/info.pb
00:11:13.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:13.903137 8293 external_mini_cluster.cc:1388] Started /tmp/dist-test-task3KSwpE/build/tsan/bin/kudu as pid 8694
00:11:13.904 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
00:11:13.904 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.8.25.124:36183
00:11:13.905 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
00:11:13.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:13.906747 8293 external_mini_cluster.cc:1326] Running /tmp/dist-test-task3KSwpE/build/tsan/bin/kudu
00:11:13.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task3KSwpE/build/tsan/bin/kudu
00:11:13.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-2/wal
00:11:13.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-2/data
00:11:13.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:11:13.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:11:13.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:11:13.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:11:13.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:11:13.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:11:13.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-2/logs
00:11:13.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-2/data/info.pb
00:11:13.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:11:13.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:11:13.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:11:13.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:11:13.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:11:13.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:11:13.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:11:13.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:11:13.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
00:11:13.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:11:13.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:11:13.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:11:13.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.25.124:36183
00:11:13.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.25.124
00:11:13.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35221
00:11:13.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/krb5kdc/kudu_127.8.25.124.keytab
00:11:13.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.25.124
00:11:13.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
00:11:13.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
00:11:13.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
00:11:13.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.25.84:33871
00:11:13.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:11:13.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:11:13.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:11:13.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:11:13.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183
00:11:13.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:11:13.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38077/jwks.json
00:11:13.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
00:11:13.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:13.910846 8772 sys_catalog.cc:263] Verifying existing consensus state
00:11:13.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260810 00:11:13.911655 8772 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
00:11:13.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:13.912119 8694 master.cc:537] Master@127.8.25.125:44265 shutting down...
00:11:13.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:13.999248 8694 master.cc:552] Master@127.8.25.125:44265 shutdown complete.
00:11:14.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
00:11:14.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:14.239048 8774 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
00:11:14.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:14.239575 8774 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:11:14.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:14.240172 8774 flags.cc:425] Enabled unsafe flag: --never_fsync=true
00:11:14.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:14.273551 8774 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
00:11:14.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:14.273768 8774 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
00:11:14.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:14.274020 8774 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
00:11:14.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:14.274286 8774 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
00:11:14.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:14.274569 8774 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
00:11:14.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:14.274766 8774 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
00:11:14.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:14.274905 8774 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38077/jwks.json
00:11:14.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:14.318123 8774 master_runner.cc:386] Master server non-default flags:
00:11:14.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:11:14.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.25.84:33871
00:11:14.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:11:14.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:11:14.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-2/data
00:11:14.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-2/wal
00:11:14.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
00:11:14.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:11:14.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183
00:11:14.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/krb5kdc/kudu_127.8.25.124.keytab
00:11:14.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.25.124
00:11:14.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:11:14.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:11:14.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:11:14.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.25.124:36183
00:11:14.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:11:14.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:11:14.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:11:14.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38077/jwks.json
00:11:14.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
00:11:14.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
00:11:14.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
00:11:14.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:11:14.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:11:14.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-2/data/info.pb
00:11:14.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.25.124
00:11:14.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35221
00:11:14.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:11:14.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:11:14.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:11:14.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:11:14.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:11:14.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-2/logs
00:11:14.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:11:14.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:11:14.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
00:11:14.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
00:11:14.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
00:11:14.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
00:11:14.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:11:14.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 23:15:58 UTC on ccycloud.quasar-ddmxrv.root.comops.site
00:11:14.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4576820
00:11:14.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
00:11:14.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:14.319480 8774 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:11:14.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:14.320796 8774 file_cache.cc:493] Constructed file cache file cache with capacity 419430
00:11:14.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:14.340433 8781 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:11:15.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:14.342904 8774 server_base.cc:1034] running on GCE node
00:11:15.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:14.340667 8783 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:11:15.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:14.340423 8780 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:11:15.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:15.644646 8774 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:11:15.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:15.647635 8774 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:11:15.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:15.649072 8774 hybrid_clock.cc:648] HybridClock initialized: now 1786320675649000 us; error 85 us; skew 500 ppm
00:11:15.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:15.650115 8774 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:11:15.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 10 00:11:15 dist-test-slave-cbbz krb5kdc[8306](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786320675, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.25.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
00:11:15.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:15.656586 8774 init.cc:377] Logged in from keytab as kudu/127.8.25.124@KRBTEST.COM (short username kudu)
00:11:15.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:15.661338 8774 webserver.cc:460] Webserver started at http://127.8.25.124:35221/ using document root <none> and password file <none>
00:11:15.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:15.662210 8774 fs_manager.cc:359] Metadata directory not provided
00:11:15.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:15.662410 8774 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:11:15.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:15.670831 8774 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.006s sys 0.000s
00:11:15.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:15.675632 8792 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
00:11:15.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:15.676709 8774 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
00:11:15.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:15.676977 8774 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-2/data,/tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-2/wal
00:11:15.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "69741f4e09d34174a4da10c0d0ccac0f"
00:11:15.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-10 00:11:09 on dist-test-slave-cbbz"
00:11:15.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:15.679325 8774 fs_report.cc:383] FS layout report
00:11:15.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:11:15.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-2/wal
00:11:15.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-2/wal
00:11:15.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-2/data/data
00:11:15.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:11:15.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:11:15.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:11:15.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:11:15.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:11:15.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:11:15.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:11:15.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:11:15.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:11:15.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:11:15.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:11:15.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:15.728204 8774 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
00:11:15.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:15.735750 8774 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:11:15.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:15.736356 8774 kserver.cc:163] Server-wide thread pool size limit: 3276
00:11:15.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:15.817095 8774 rpc_server.cc:307] RPC server started. Bound to: 127.8.25.124:36183
00:11:15.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:15.817253 8849 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.8.25.124:36183 every 8 connection(s)
00:11:15.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:15.819934 8774 server_base.cc:1166] Dumped server information to /tmp/dist-test-task3KSwpE/test-tmp/mini-kudu-cluster6200734772338387484/master-2/data/info.pb
00:11:15.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:15.822026 8293 external_mini_cluster.cc:1388] Started /tmp/dist-test-task3KSwpE/build/tsan/bin/kudu as pid 8774
00:11:15.822 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
00:11:15.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:15.834477 8850 sys_catalog.cc:263] Verifying existing consensus state
00:11:15.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:15.842469 8850 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f: Bootstrap starting.
00:11:15.885 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.8.25.126:34221(127.8.25.126:34221)] connecting to peer
00:11:15.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:15.887655 8850 log.cc:824] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f: Log is configured to *not* fsync() on all Append() calls
00:11:15.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:15.893023 8850 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
00:11:15.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:15.893930 8850 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f: Bootstrap complete.
00:11:15.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:15.926246 8850 raft_consensus.cc:348] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2b03379f7f604a7d9bec7fac55417f0f" member_type: VOTER last_known_addr { host: "127.8.25.126" port: 34221 } } peers { permanent_uuid: "88c642733e5b44adaefc882d98c8d199" member_type: VOTER last_known_addr { host: "127.8.25.125" port: 44265 } } peers { permanent_uuid: "69741f4e09d34174a4da10c0d0ccac0f" member_type: VOTER last_known_addr { host: "127.8.25.124" port: 36183 } }
00:11:15.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:15.928732 8850 raft_consensus.cc:374] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
00:11:15.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:15.929018 8850 raft_consensus.cc:729] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 69741f4e09d34174a4da10c0d0ccac0f, State: Initialized, Role: FOLLOWER
00:11:15.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:15.930014 8850 consensus_queue.cc:260] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2b03379f7f604a7d9bec7fac55417f0f" member_type: VOTER last_known_addr { host: "127.8.25.126" port: 34221 } } peers { permanent_uuid: "88c642733e5b44adaefc882d98c8d199" member_type: VOTER last_known_addr { host: "127.8.25.125" port: 44265 } } peers { permanent_uuid: "69741f4e09d34174a4da10c0d0ccac0f" member_type: VOTER last_known_addr { host: "127.8.25.124" port: 36183 } }
00:11:15.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:15.933816 8854 sys_catalog.cc:455] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2b03379f7f604a7d9bec7fac55417f0f" member_type: VOTER last_known_addr { host: "127.8.25.126" port: 34221 } } peers { permanent_uuid: "88c642733e5b44adaefc882d98c8d199" member_type: VOTER last_known_addr { host: "127.8.25.125" port: 44265 } } peers { permanent_uuid: "69741f4e09d34174a4da10c0d0ccac0f" member_type: VOTER last_known_addr { host: "127.8.25.124" port: 36183 } } }
00:11:15.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:15.934717 8854 sys_catalog.cc:458] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f [sys.catalog]: This master's current role is: FOLLOWER
00:11:15.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:15.935250 8850 sys_catalog.cc:564] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f [sys.catalog]: configured and running, proceeding with master startup.
00:11:15.954 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.8.25.125:44265(127.8.25.125:44265)] connecting to peer
00:11:15.957 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.8.25.124:36183(127.8.25.124:36183)] connecting to peer
00:11:15.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:15.975432 8854 raft_consensus.cc:482] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
00:11:15.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:15.976042 8854 raft_consensus.cc:504] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2b03379f7f604a7d9bec7fac55417f0f" member_type: VOTER last_known_addr { host: "127.8.25.126" port: 34221 } } peers { permanent_uuid: "88c642733e5b44adaefc882d98c8d199" member_type: VOTER last_known_addr { host: "127.8.25.125" port: 44265 } } peers { permanent_uuid: "69741f4e09d34174a4da10c0d0ccac0f" member_type: VOTER last_known_addr { host: "127.8.25.124" port: 36183 } }
00:11:15.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:15.982734 8854 leader_election.cc:288] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 2b03379f7f604a7d9bec7fac55417f0f (127.8.25.126:34221), 88c642733e5b44adaefc882d98c8d199 (127.8.25.125:44265)
00:11:15.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:15.989504 8795 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.25.125:44265: connect: Connection refused (error 111)
00:11:16.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:15.992668 8798 tablet.cc:2369] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f: Can't schedule compaction. Clean time has not been advanced past its initial value.
00:11:16.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:16.013777 8795 leader_election.cc:334] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 88c642733e5b44adaefc882d98c8d199 (127.8.25.125:44265): Network error: Client connection negotiation failed: client connection to 127.8.25.125:44265: connect: Connection refused (error 111)
00:11:16.029 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:16.030 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.25.125:44265(127.8.25.125:44265)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:16.031 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:16.031 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.25.126:34221(127.8.25.126:34221)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:16.033 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.25.126:34221: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:16.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:16.033051 8794 leader_election.cc:334] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 2b03379f7f604a7d9bec7fac55417f0f (127.8.25.126:34221): Network error: Client connection negotiation failed: client connection to 127.8.25.126:34221: connect: Connection refused (error 111)
00:11:16.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:16.033631 8794 leader_election.cc:302] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 69741f4e09d34174a4da10c0d0ccac0f; no voters: 2b03379f7f604a7d9bec7fac55417f0f, 88c642733e5b44adaefc882d98c8d199
00:11:16.035 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.8.25.124:36183(127.8.25.124:36183)] Successfully connected to peer
00:11:16.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:16.035373 8854 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
00:11:16.037 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.25.125:44265: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:16.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:16.033152 8868 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:11:16.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:16.042819 8868 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:11:16.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:16.020102 8774 master_runner.cc:186] Error getting master registration for 127.8.25.126:34221: Network error: Client connection negotiation failed: client connection to 127.8.25.126:34221: connect: Connection refused (error 111),
00:11:16.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:16.051649 8774 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
00:11:16.117 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
00:11:16.128 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.8.25.124
00:11:16.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 10 00:11:16 dist-test-slave-cbbz krb5kdc[8306](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786320670, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.8.25.124@KRBTEST.COM
00:11:16.390 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x2d925312, L:/127.0.0.1:53696 - R:/127.8.25.124:36183] using SASL/GSSAPI
00:11:16.406 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183; will retry
00:11:16.430 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.25.126:34221(127.8.25.126:34221)] connecting to peer
00:11:16.433 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.25.125:44265(127.8.25.125:44265)] connecting to peer
00:11:16.435 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:16.435 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.25.126:34221(127.8.25.126:34221)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:16.436 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.25.126:34221: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:16.440 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:16.440 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.25.125:44265(127.8.25.125:44265)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:16.440 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.25.125:44265: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:16.442 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183; will retry
00:11:16.448 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.25.126:34221(127.8.25.126:34221)] connecting to peer
00:11:16.450 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:16.450 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.25.125:44265(127.8.25.125:44265)] connecting to peer
00:11:16.451 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.25.126:34221(127.8.25.126:34221)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:16.451 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.25.126:34221: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:16.453 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:16.454 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.25.125:44265(127.8.25.125:44265)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:16.454 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.25.125:44265: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:16.456 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183; will retry
00:11:16.469 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.25.126:34221(127.8.25.126:34221)] connecting to peer
00:11:16.470 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.25.125:44265(127.8.25.125:44265)] connecting to peer
00:11:16.471 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:16.472 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.25.126:34221(127.8.25.126:34221)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:16.473 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.25.126:34221: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:16.475 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:16.475 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.25.125:44265(127.8.25.125:44265)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:16.476 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.25.125:44265: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:16.480 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183; will retry
00:11:16.509 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.25.126:34221(127.8.25.126:34221)] connecting to peer
00:11:16.511 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.25.125:44265(127.8.25.125:44265)] connecting to peer
00:11:16.512 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:16.512 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.25.126:34221(127.8.25.126:34221)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:16.513 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.25.126:34221: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:16.516 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:16.516 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.25.125:44265(127.8.25.125:44265)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:16.517 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.25.125:44265: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:16.518 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183; will retry
00:11:16.548 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.25.126:34221(127.8.25.126:34221)] connecting to peer
00:11:16.550 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.25.125:44265(127.8.25.125:44265)] connecting to peer
00:11:16.552 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:16.552 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.25.126:34221(127.8.25.126:34221)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:16.552 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.25.126:34221: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:16.555 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:16.555 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.25.125:44265(127.8.25.125:44265)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:16.555 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.25.125:44265: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:16.560 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183; will retry
00:11:16.609 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.25.126:34221(127.8.25.126:34221)] connecting to peer
00:11:16.611 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.25.125:44265(127.8.25.125:44265)] connecting to peer
00:11:16.613 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:16.613 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.25.126:34221(127.8.25.126:34221)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:16.614 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.25.126:34221: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:16.615 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:16.616 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.25.125:44265(127.8.25.125:44265)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:16.616 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.25.125:44265: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:16.618 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183; will retry
00:11:16.689 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.25.126:34221(127.8.25.126:34221)] connecting to peer
00:11:16.692 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.25.125:44265(127.8.25.125:44265)] connecting to peer
00:11:16.692 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:16.693 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.25.126:34221(127.8.25.126:34221)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:16.693 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.25.126:34221: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:16.695 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:16.695 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.25.125:44265(127.8.25.125:44265)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:16.696 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.25.125:44265: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:16.698 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183; will retry
00:11:16.789 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.25.126:34221(127.8.25.126:34221)] connecting to peer
00:11:16.790 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.25.125:44265(127.8.25.125:44265)] connecting to peer
00:11:16.791 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:16.791 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.25.126:34221(127.8.25.126:34221)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:16.793 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.25.126:34221: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:16.795 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:16.795 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.25.125:44265(127.8.25.125:44265)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:16.799 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.25.125:44265: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:16.801 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183; will retry
00:11:16.849 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.25.126:34221(127.8.25.126:34221)] connecting to peer
00:11:16.850 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.25.125:44265(127.8.25.125:44265)] connecting to peer
00:11:16.853 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:16.853 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.25.126:34221(127.8.25.126:34221)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:16.854 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.25.126:34221: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:16.855 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:16.855 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.25.125:44265(127.8.25.125:44265)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:16.856 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.25.125:44265: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:16.858 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183; will retry
00:11:17.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:17.046835 8868 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:11:17.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:17.047128 8868 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:11:17.431 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.25.126:34221(127.8.25.126:34221)] connecting to peer
00:11:17.433 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.25.125:44265(127.8.25.125:44265)] connecting to peer
00:11:17.435 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:17.436 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.25.126:34221(127.8.25.126:34221)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:17.437 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:17.437 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.25.126:34221: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:17.437 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.25.125:44265(127.8.25.125:44265)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:17.437 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.25.125:44265: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:17.440 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183; will retry
00:11:17.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:17.835230 8877 raft_consensus.cc:482] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
00:11:17.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:17.835703 8877 raft_consensus.cc:504] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2b03379f7f604a7d9bec7fac55417f0f" member_type: VOTER last_known_addr { host: "127.8.25.126" port: 34221 } } peers { permanent_uuid: "88c642733e5b44adaefc882d98c8d199" member_type: VOTER last_known_addr { host: "127.8.25.125" port: 44265 } } peers { permanent_uuid: "69741f4e09d34174a4da10c0d0ccac0f" member_type: VOTER last_known_addr { host: "127.8.25.124" port: 36183 } }
00:11:17.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:17.837658 8877 leader_election.cc:288] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 2b03379f7f604a7d9bec7fac55417f0f (127.8.25.126:34221), 88c642733e5b44adaefc882d98c8d199 (127.8.25.125:44265)
00:11:17.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:17.845252 8794 leader_election.cc:334] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 2b03379f7f604a7d9bec7fac55417f0f (127.8.25.126:34221): Network error: Client connection negotiation failed: client connection to 127.8.25.126:34221: connect: Connection refused (error 111)
00:11:17.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:17.846131 8795 leader_election.cc:334] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 88c642733e5b44adaefc882d98c8d199 (127.8.25.125:44265): Network error: Client connection negotiation failed: client connection to 127.8.25.125:44265: connect: Connection refused (error 111)
00:11:17.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:17.846627 8795 leader_election.cc:302] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 69741f4e09d34174a4da10c0d0ccac0f; no voters: 2b03379f7f604a7d9bec7fac55417f0f, 88c642733e5b44adaefc882d98c8d199
00:11:17.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:17.847474 8877 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
00:11:18.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:18.050348 8868 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:11:18.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:18.050607 8868 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:11:19.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:19.053766 8868 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:11:19.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:19.054029 8868 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:11:19.109 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.25.126:34221(127.8.25.126:34221)] connecting to peer
00:11:19.110 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.25.125:44265(127.8.25.125:44265)] connecting to peer
00:11:19.111 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:19.112 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:19.112 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.25.126:34221(127.8.25.126:34221)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:19.112 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.25.125:44265(127.8.25.125:44265)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:19.113 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.25.125:44265: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:19.113 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.25.126:34221: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:19.116 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183; will retry
00:11:19.729 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.25.126:34221(127.8.25.126:34221)] connecting to peer
00:11:19.730 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.25.125:44265(127.8.25.125:44265)] connecting to peer
00:11:19.731 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:19.731 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.25.126:34221(127.8.25.126:34221)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:19.731 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.25.126:34221: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:19.733 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:19.733 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.25.125:44265(127.8.25.125:44265)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:19.733 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.25.125:44265: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:19.735 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183; will retry
00:11:19.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:19.789381 8884 raft_consensus.cc:482] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
00:11:19.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:19.789798 8884 raft_consensus.cc:504] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2b03379f7f604a7d9bec7fac55417f0f" member_type: VOTER last_known_addr { host: "127.8.25.126" port: 34221 } } peers { permanent_uuid: "88c642733e5b44adaefc882d98c8d199" member_type: VOTER last_known_addr { host: "127.8.25.125" port: 44265 } } peers { permanent_uuid: "69741f4e09d34174a4da10c0d0ccac0f" member_type: VOTER last_known_addr { host: "127.8.25.124" port: 36183 } }
00:11:19.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:19.791369 8884 leader_election.cc:288] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 2b03379f7f604a7d9bec7fac55417f0f (127.8.25.126:34221), 88c642733e5b44adaefc882d98c8d199 (127.8.25.125:44265)
00:11:19.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:19.796519 8795 leader_election.cc:334] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 88c642733e5b44adaefc882d98c8d199 (127.8.25.125:44265): Network error: Client connection negotiation failed: client connection to 127.8.25.125:44265: connect: Connection refused (error 111)
00:11:19.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:19.796927 8794 leader_election.cc:334] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 2b03379f7f604a7d9bec7fac55417f0f (127.8.25.126:34221): Network error: Client connection negotiation failed: client connection to 127.8.25.126:34221: connect: Connection refused (error 111)
00:11:19.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:19.797260 8794 leader_election.cc:302] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 69741f4e09d34174a4da10c0d0ccac0f; no voters: 2b03379f7f604a7d9bec7fac55417f0f, 88c642733e5b44adaefc882d98c8d199
00:11:19.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:19.797755 8884 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
00:11:20.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:20.057005 8868 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:11:20.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:20.057305 8868 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:11:21.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:21.054364 8774 master_runner.cc:186] Error getting master registration for 127.8.25.126:34221: Network error: Client connection negotiation failed: client connection to 127.8.25.126:34221: connect: Connection refused (error 111),
00:11:21.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:21.054760 8774 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
00:11:21.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:21.059938 8868 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:11:21.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:21.060236 8868 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:11:22.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:22.063115 8868 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:11:22.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:22.063367 8868 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:11:22.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:22.349854 8892 raft_consensus.cc:482] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
00:11:22.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:22.350291 8892 raft_consensus.cc:504] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2b03379f7f604a7d9bec7fac55417f0f" member_type: VOTER last_known_addr { host: "127.8.25.126" port: 34221 } } peers { permanent_uuid: "88c642733e5b44adaefc882d98c8d199" member_type: VOTER last_known_addr { host: "127.8.25.125" port: 44265 } } peers { permanent_uuid: "69741f4e09d34174a4da10c0d0ccac0f" member_type: VOTER last_known_addr { host: "127.8.25.124" port: 36183 } }
00:11:22.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:22.351929 8892 leader_election.cc:288] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 2b03379f7f604a7d9bec7fac55417f0f (127.8.25.126:34221), 88c642733e5b44adaefc882d98c8d199 (127.8.25.125:44265)
00:11:22.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:22.353790 8794 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.25.126:34221: connect: Connection refused (error 111) [suppressed 5 similar messages]
00:11:22.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:22.356122 8795 leader_election.cc:334] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 88c642733e5b44adaefc882d98c8d199 (127.8.25.125:44265): Network error: Client connection negotiation failed: client connection to 127.8.25.125:44265: connect: Connection refused (error 111)
00:11:22.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:22.356524 8794 leader_election.cc:334] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 2b03379f7f604a7d9bec7fac55417f0f (127.8.25.126:34221): Network error: Client connection negotiation failed: client connection to 127.8.25.126:34221: connect: Connection refused (error 111)
00:11:22.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:22.356845 8794 leader_election.cc:302] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 69741f4e09d34174a4da10c0d0ccac0f; no voters: 2b03379f7f604a7d9bec7fac55417f0f, 88c642733e5b44adaefc882d98c8d199
00:11:22.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:22.357362 8892 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
00:11:22.589 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.25.126:34221(127.8.25.126:34221)] connecting to peer
00:11:22.590 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.25.125:44265(127.8.25.125:44265)] connecting to peer
00:11:22.592 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:22.592 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.25.126:34221(127.8.25.126:34221)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:22.592 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.25.126:34221: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:22.593 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:22.593 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.25.125:44265(127.8.25.125:44265)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:22.593 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.25.125:44265: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:22.595 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183; will retry
00:11:23.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:23.066331 8868 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:11:23.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:23.066581 8868 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:11:24.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:24.069376 8868 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:11:24.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:24.069629 8868 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:11:25.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:25.073303 8868 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:11:25.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:25.073648 8868 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:11:25.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 10 00:11:25 dist-test-slave-cbbz krb5kdc[8306](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786320685, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.25.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
00:11:25.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:25.660501 8788 init.cc:295] Successfully reacquired a new kerberos TGT
00:11:25.669 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.25.126:34221(127.8.25.126:34221)] connecting to peer
00:11:25.670 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.25.125:44265(127.8.25.125:44265)] connecting to peer
00:11:25.672 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:25.672 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.25.126:34221(127.8.25.126:34221)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:25.672 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.25.126:34221: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:25.673 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:25.674 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.25.125:44265(127.8.25.125:44265)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:25.674 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.25.125:44265: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:25.676 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183; will retry
00:11:26.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:26.058121 8774 master_runner.cc:186] Error getting master registration for 127.8.25.126:34221: Network error: Client connection negotiation failed: client connection to 127.8.25.126:34221: connect: Connection refused (error 111),
00:11:26.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:26.058527 8774 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
00:11:26.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:26.076792 8868 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:11:26.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:26.077051 8868 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:11:27.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:27.002297 8900 raft_consensus.cc:482] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
00:11:27.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:27.002876 8900 raft_consensus.cc:504] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2b03379f7f604a7d9bec7fac55417f0f" member_type: VOTER last_known_addr { host: "127.8.25.126" port: 34221 } } peers { permanent_uuid: "88c642733e5b44adaefc882d98c8d199" member_type: VOTER last_known_addr { host: "127.8.25.125" port: 44265 } } peers { permanent_uuid: "69741f4e09d34174a4da10c0d0ccac0f" member_type: VOTER last_known_addr { host: "127.8.25.124" port: 36183 } }
00:11:27.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:27.004808 8900 leader_election.cc:288] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 2b03379f7f604a7d9bec7fac55417f0f (127.8.25.126:34221), 88c642733e5b44adaefc882d98c8d199 (127.8.25.125:44265)
00:11:27.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:27.009940 8794 leader_election.cc:334] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 2b03379f7f604a7d9bec7fac55417f0f (127.8.25.126:34221): Network error: Client connection negotiation failed: client connection to 127.8.25.126:34221: connect: Connection refused (error 111)
00:11:27.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:27.010591 8795 leader_election.cc:334] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 88c642733e5b44adaefc882d98c8d199 (127.8.25.125:44265): Network error: Client connection negotiation failed: client connection to 127.8.25.125:44265: connect: Connection refused (error 111)
00:11:27.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:27.010913 8795 leader_election.cc:302] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 69741f4e09d34174a4da10c0d0ccac0f; no voters: 2b03379f7f604a7d9bec7fac55417f0f, 88c642733e5b44adaefc882d98c8d199
00:11:27.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:27.011485 8900 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
00:11:27.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:27.080440 8868 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:11:27.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:27.080693 8868 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:11:28.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:28.083699 8868 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:11:28.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:28.084003 8868 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:11:28.749 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.25.126:34221(127.8.25.126:34221)] connecting to peer
00:11:28.751 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.25.125:44265(127.8.25.125:44265)] connecting to peer
00:11:28.751 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:28.752 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.25.126:34221(127.8.25.126:34221)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:28.753 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:28.753 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.25.125:44265(127.8.25.125:44265)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:28.753 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.25.125:44265: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:28.754 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.25.126:34221: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:28.756 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183; will retry
00:11:29.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:29.086927 8868 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:11:29.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:29.087188 8868 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:11:29.609 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.25.126:34221(127.8.25.126:34221)] connecting to peer
00:11:29.611 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.25.125:44265(127.8.25.125:44265)] connecting to peer
00:11:29.612 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:29.613 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.25.126:34221(127.8.25.126:34221)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:29.613 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.25.126:34221: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:29.614 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:29.614 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.25.125:44265(127.8.25.125:44265)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:29.614 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.25.125:44265: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:29.616 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183; will retry
00:11:30.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:30.090188 8868 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:11:30.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:30.090449 8868 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:11:31.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:31.061110 8774 master_runner.cc:186] Error getting master registration for 127.8.25.126:34221: Network error: Client connection negotiation failed: client connection to 127.8.25.126:34221: connect: Connection refused (error 111),
00:11:31.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:31.061460 8774 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
00:11:31.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:31.093338 8868 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:11:31.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:31.093583 8868 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:11:31.689 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.25.126:34221(127.8.25.126:34221)] connecting to peer
00:11:31.690 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.25.125:44265(127.8.25.125:44265)] connecting to peer
00:11:31.692 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:31.692 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.25.126:34221(127.8.25.126:34221)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:31.692 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.25.126:34221: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:31.693 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:31.693 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.25.125:44265(127.8.25.125:44265)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:31.693 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.25.125:44265: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:31.695 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183; will retry
00:11:32.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:32.096421 8868 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:11:32.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:32.096675 8868 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:11:33.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:33.099606 8868 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:11:33.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:33.099855 8868 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:11:34.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:34.103171 8868 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:11:34.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:34.103497 8868 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:11:35.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:35.107177 8868 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:11:35.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:35.107499 8868 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:11:35.469 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.25.126:34221(127.8.25.126:34221)] connecting to peer
00:11:35.470 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.25.125:44265(127.8.25.125:44265)] connecting to peer
00:11:35.471 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:35.472 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.25.126:34221(127.8.25.126:34221)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:35.472 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.25.126:34221: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:35.473 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:35.473 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.25.125:44265(127.8.25.125:44265)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:35.474 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.25.125:44265: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:35.476 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183; will retry
00:11:35.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:35.581808 8917 raft_consensus.cc:482] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
00:11:35.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:35.582350 8917 raft_consensus.cc:504] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2b03379f7f604a7d9bec7fac55417f0f" member_type: VOTER last_known_addr { host: "127.8.25.126" port: 34221 } } peers { permanent_uuid: "88c642733e5b44adaefc882d98c8d199" member_type: VOTER last_known_addr { host: "127.8.25.125" port: 44265 } } peers { permanent_uuid: "69741f4e09d34174a4da10c0d0ccac0f" member_type: VOTER last_known_addr { host: "127.8.25.124" port: 36183 } }
00:11:35.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:35.584060 8917 leader_election.cc:288] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 2b03379f7f604a7d9bec7fac55417f0f (127.8.25.126:34221), 88c642733e5b44adaefc882d98c8d199 (127.8.25.125:44265)
00:11:35.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:35.587085 8794 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.25.126:34221: connect: Connection refused (error 111) [suppressed 3 similar messages]
00:11:35.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:35.590082 8795 leader_election.cc:334] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 88c642733e5b44adaefc882d98c8d199 (127.8.25.125:44265): Network error: Client connection negotiation failed: client connection to 127.8.25.125:44265: connect: Connection refused (error 111)
00:11:35.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:35.590493 8794 leader_election.cc:334] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 2b03379f7f604a7d9bec7fac55417f0f (127.8.25.126:34221): Network error: Client connection negotiation failed: client connection to 127.8.25.126:34221: connect: Connection refused (error 111)
00:11:35.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:35.590806 8794 leader_election.cc:302] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 69741f4e09d34174a4da10c0d0ccac0f; no voters: 2b03379f7f604a7d9bec7fac55417f0f, 88c642733e5b44adaefc882d98c8d199
00:11:35.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:35.591343 8917 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
00:11:35.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 10 00:11:35 dist-test-slave-cbbz krb5kdc[8306](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786320695, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.25.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
00:11:35.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:35.663044 8788 init.cc:295] Successfully reacquired a new kerberos TGT
00:11:36.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:36.063784 8774 master_runner.cc:186] Error getting master registration for 127.8.25.126:34221: Network error: Client connection negotiation failed: client connection to 127.8.25.126:34221: connect: Connection refused (error 111),
00:11:36.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:36.064338 8774 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
00:11:36.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:36.110533 8868 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:11:36.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:36.110793 8868 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:11:37.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:37.113777 8868 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:11:37.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:37.114071 8868 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:11:38.108 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.25.126:34221(127.8.25.126:34221)] connecting to peer
00:11:38.110 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.25.125:44265(127.8.25.125:44265)] connecting to peer
00:11:38.111 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:38.111 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.25.126:34221(127.8.25.126:34221)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:38.112 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.25.126:34221: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:38.114 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:38.114 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.25.125:44265(127.8.25.125:44265)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:38.114 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.25.125:44265: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:38.116 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183; will retry
00:11:38.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:38.117440 8868 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:11:38.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:38.117682 8868 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:11:39.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:39.120589 8868 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:11:39.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:39.120848 8868 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:11:40.029 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.25.126:34221(127.8.25.126:34221)] connecting to peer
00:11:40.030 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.25.125:44265(127.8.25.125:44265)] connecting to peer
00:11:40.030 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:40.030 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.25.126:34221(127.8.25.126:34221)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:40.030 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.25.126:34221: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:40.032 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:40.032 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.25.125:44265(127.8.25.125:44265)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:40.032 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.25.125:44265: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:40.034 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183; will retry
00:11:40.069 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.25.126:34221(127.8.25.126:34221)] connecting to peer
00:11:40.070 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.25.125:44265(127.8.25.125:44265)] connecting to peer
00:11:40.070 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:40.070 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.25.126:34221(127.8.25.126:34221)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:40.071 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.25.126:34221: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:40.072 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:40.072 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.25.125:44265(127.8.25.125:44265)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:40.072 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.25.125:44265: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:40.074 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183; will retry
00:11:40.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:40.123584 8868 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:11:40.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:40.123834 8868 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:11:41.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:41.066771 8774 master_runner.cc:186] Error getting master registration for 127.8.25.126:34221: Network error: Client connection negotiation failed: client connection to 127.8.25.126:34221: connect: Connection refused (error 111),
00:11:41.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:41.067101 8774 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
00:11:41.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:41.126720 8868 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:11:41.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:41.126977 8868 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:11:41.149 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.25.126:34221(127.8.25.126:34221)] connecting to peer
00:11:41.149 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.25.125:44265(127.8.25.125:44265)] connecting to peer
00:11:41.150 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:41.150 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.25.126:34221(127.8.25.126:34221)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:41.150 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.25.126:34221: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:41.151 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:41.151 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.25.125:44265(127.8.25.125:44265)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:41.151 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.25.125:44265: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:41.153 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183; will retry
00:11:42.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:42.129834 8868 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:11:42.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:42.130100 8868 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:11:43.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:43.133179 8868 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 69741f4e09d34174a4da10c0d0ccac0f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:11:43.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 00:11:43.133561 8868 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:11:43.849 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.25.126:34221(127.8.25.126:34221)] connecting to peer
00:11:43.850 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.25.125:44265(127.8.25.125:44265)] connecting to peer
00:11:43.851 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:43.851 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.25.126:34221(127.8.25.126:34221)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:43.851 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.25.126:34221: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221
00:11:43.852 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:43.852 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.25.125:44265(127.8.25.125:44265)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:43.852 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.25.125:44265: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
00:11:43.855 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183; will retry
00:11:43.859 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=28028), Traces: [0ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.126:34221, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.125:44265, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.124:36183, [197ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.126:34221: Network error: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221, [196ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.125:44265: Network error: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [568ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.124:36183: OK, [572ms] delaying RPC due to: Service unavailable: Master config (127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [595ms] refreshing cache from master, [595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.126:34221, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.125:44265, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.124:36183, [601ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.126:34221: Network error: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221, [606ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.125:44265: Network error: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [607ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.124:36183: OK, [608ms] delaying RPC due to: Service unavailable: Master config (127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [614ms] refreshing cache from master, [614ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.126:34221, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.125:44265, [617ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.126:34221: Network error: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.124:36183, [620ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.125:44265: Network error: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [621ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.124:36183: OK, [622ms] delaying RPC due to: Service unavailable: Master config (127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [634ms] refreshing cache from master, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.126:34221, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.125:44265, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.124:36183, [638ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.126:34221: Network error: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221, [641ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.125:44265: Network error: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [644ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.124:36183: OK, [646ms] delaying RPC due to: Service unavailable: Master config (127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [674ms] refreshing cache from master, [675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.126:34221, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.125:44265, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.124:36183, [679ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.126:34221: Network error: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221, [682ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.125:44265: Network error: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [684ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.124:36183: OK, [684ms] delaying RPC due to: Service unavailable: Master config (127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [714ms] refreshing cache from master, [714ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.126:34221, [715ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.125:44265, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.124:36183, [718ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.126:34221: Network error: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221, [721ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.125:44265: Network error: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [725ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.124:36183: OK, [726ms] delaying RPC due to: Service unavailable: Master config (127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [775ms] refreshing cache from master, [775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.126:34221, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.125:44265, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.124:36183, [779ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.126:34221: Network error: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221, [782ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.125:44265: Network error: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [784ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.124:36183: OK, [784ms] delaying RPC due to: Service unavailable: Master config (127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [855ms] refreshing cache from master, [855ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.126:34221, [857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.125:44265, [858ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.124:36183, [859ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.126:34221: Network error: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221, [861ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.125:44265: Network error: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [863ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.124:36183: OK, [865ms] delaying RPC due to: Service unavailable: Master config (127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [954ms] refreshing cache from master, [954ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.126:34221, [956ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.125:44265, [957ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.124:36183, [958ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.126:34221: Network error: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221, [961ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.125:44265: Network error: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [967ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.124:36183: OK, [968ms] delaying RPC due to: Service unavailable: Master config (127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [1014ms] refreshing cache from master, [1015ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.126:34221, [1016ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.125:44265, [1018ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.124:36183, [1019ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.126:34221: Network error: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221, [1022ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.125:44265: Network error: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [1023ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.124:36183: OK, [1024ms] delaying RPC due to: Service unavailable: Master config (127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [1597ms] refreshing cache from master, [1597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.126:34221, [1599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.125:44265, [1600ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.124:36183, [1603ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.126:34221: Network error: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221, [1603ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.125:44265: Network error: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [1605ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.124:36183: OK, [1606ms] delaying RPC due to: Service unavailable: Master config (127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [3274ms] refreshing cache from master, [3275ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.126:34221, [3276ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.125:44265, [3277ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.124:36183, [3278ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.125:44265: Network error: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [3278ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.126:34221: Network error: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221, [3282ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.124:36183: OK, [3283ms] delaying RPC due to: Service unavailable: Master config (127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221, [3894ms] refreshing cache from master, [3894ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.126:34221, [3895ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.125:44265, [3896ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.124:36183, [3897ms] too many traces: truncated at 100 traces, deferred=Deferred@559077857(state=PENDING, result=null, callback=(continuation of Deferred@1471534627 after retry RPC after error@1495606074) -> (continuation of Deferred@1531322372 after retry RPC after error@1827436380) -> (continuation of Deferred@361865260 after retry RPC after error@1709014359) -> (continuation of Deferred@259074024 after retry RPC after error@11069389) -> (continuation of Deferred@404793140 after retry RPC after error@370755234) -> (continuation of Deferred@282164718 after retry RPC after error@1468032694) -> (continuation of Deferred@1241801333 after retry RPC after error@376705473) -> (continuation of Deferred@231809260 after retry RPC after error@1504727606) -> (continuation of Deferred@818840767 after retry RPC after error@109207934) -> (continuation of Deferred@38856450 after retry RPC after error@1504326812) -> (continuation of Deferred@486910058 after retry RPC after error@135901989) -> (continuation of Deferred@225721811 after retry RPC after error@91420213) -> (continuation of Deferred@889477129 after retry RPC after error@1891718423) -> (continuation of Deferred@1551332473 after retry RPC after error@1319563932) -> (continuation of Deferred@1509244753 after retry RPC after error@593932437) -> (continuation of Deferred@1154862140 after retry RPC after error@1025438760) -> (continuation of Deferred@1306610541 after retry RPC after error@1536534620) -> (continuation of Deferred@827013950 after retry RPC after error@2067674763) -> (continuation of Deferred@1011069090 after retry RPC after error@1247946814) -> (continuation of Deferred@580413575 after retry RPC after error@672700465) -> (continuation of Deferred@167766832 after retry RPC after error@1376190953) -> (continuation of Deferred@867862398 after retry RPC after error@1594837032) -> (continuation of Deferred@702516882 after retry RPC after error@1136511497), errback=(continuation of Deferred@1471534627 after retry RPC after error@1495606074) -> (continuation of Deferred@1531322372 after retry RPC after error@1827436380) -> (continuation of Deferred@361865260 after retry RPC after error@1709014359) -> (continuation of Deferred@259074024 after retry RPC after error@11069389) -> (continuation of Deferred@404793140 after retry RPC after error@370755234) -> (continuation of Deferred@282164718 after retry RPC after error@1468032694) -> (continuation of Deferred@1241801333 after retry RPC after error@376705473) -> (continuation of Deferred@231809260 after retry RPC after error@1504727606) -> (continuation of Deferred@818840767 after retry RPC after error@109207934) -> (continuation of Deferred@38856450 after retry RPC after error@1504326812) -> (continuation of Deferred@486910058 after retry RPC after error@135901989) -> (continuation of Deferred@225721811 after retry RPC after error@91420213) -> (continuation of Deferred@889477129 after retry RPC after error@1891718423) -> (continuation of Deferred@1551332473 after retry RPC after error@1319563932) -> (continuation of Deferred@1509244753 after retry RPC after error@593932437) -> (continuation of Deferred@1154862140 after retry RPC after error@1025438760) -> (continuation of Deferred@1306610541 after retry RPC after error@1536534620) -> (continuation of Deferred@827013950 after retry RPC after error@2067674763) -> (continuation of Deferred@1011069090 after retry RPC after error@1247946814) -> (continuation of Deferred@580413575 after retry RPC after error@672700465) -> (continuation of Deferred@167766832 after retry RPC after error@1376190953) -> (continuation of Deferred@867862398 after retry RPC after error@1594837032) -> (continuation of Deferred@702516882 after retry RPC after error@1136511497)))
00:11:43.865 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.8.25.126:34221(127.8.25.126:34221)] disconnecting while in state TERMINATED
00:11:43.866 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.8.25.124:36183(127.8.25.124:36183)] disconnecting while in state READY
00:11:43.867 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.8.25.125:44265(127.8.25.125:44265)] disconnecting while in state TERMINATED
00:11:43.872 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
00:11:43.874 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.8.25.124:36183(127.8.25.124:36183)] handling channelInactive
00:11:43.874 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.25.124:36183(127.8.25.124:36183)] cleaning up while in state READY due to: connection closed
00:11:43.876 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
00:11:43.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:43.883507 8293 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task3KSwpE/build/tsan/bin/kudu with pid 8774
00:11:45.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-10T00:11:45Z chronyd exiting
00:11:46.122 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
00:11:46.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:46.124224 7499 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task3KSwpE/build/tsan/bin/kudu with pid 7807
00:11:46.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:46.154096 7499 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task3KSwpE/build/tsan/bin/kudu with pid 7988
00:11:46.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:46.184046 7499 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task3KSwpE/build/tsan/bin/kudu with pid 8140
00:11:46.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:46.214896 7499 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task3KSwpE/build/tsan/bin/kudu with pid 7515
00:11:46.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:46.252683 7499 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task3KSwpE/build/tsan/bin/kudu with pid 7588
00:11:46.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 00:11:46.287137 7499 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task3KSwpE/build/tsan/bin/kudu with pid 7669
00:11:47.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-10T00:11:47Z chronyd exiting
00:11:48.470 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=28028), Traces: [0ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.126:34221, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.125:44265, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.124:36183, [197ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.126:34221: Network error: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221, [196ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.125:44265: Network error: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [568ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.124:36183: OK, [572ms] delaying RPC due to: Service unavailable: Master config (127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [595ms] refreshing cache from master, [595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.126:34221, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.125:44265, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.124:36183, [601ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.126:34221: Network error: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221, [606ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.125:44265: Network error: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [607ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.124:36183: OK, [608ms] delaying RPC due to: Service unavailable: Master config (127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [614ms] refreshing cache from master, [614ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.126:34221, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.125:44265, [617ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.126:34221: Network error: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.124:36183, [620ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.125:44265: Network error: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [621ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.124:36183: OK, [622ms] delaying RPC due to: Service unavailable: Master config (127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [634ms] refreshing cache from master, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.126:34221, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.125:44265, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.124:36183, [638ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.126:34221: Network error: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221, [641ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.125:44265: Network error: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [644ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.124:36183: OK, [646ms] delaying RPC due to: Service unavailable: Master config (127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [674ms] refreshing cache from master, [675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.126:34221, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.125:44265, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.124:36183, [679ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.126:34221: Network error: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221, [682ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.125:44265: Network error: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [684ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.124:36183: OK, [684ms] delaying RPC due to: Service unavailable: Master config (127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [714ms] refreshing cache from master, [714ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.126:34221, [715ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.125:44265, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.124:36183, [718ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.126:34221: Network error: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221, [721ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.125:44265: Network error: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [725ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.124:36183: OK, [726ms] delaying RPC due to: Service unavailable: Master config (127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [775ms] refreshing cache from master, [775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.126:34221, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.125:44265, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.124:36183, [779ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.126:34221: Network error: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221, [782ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.125:44265: Network error: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [784ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.124:36183: OK, [784ms] delaying RPC due to: Service unavailable: Master config (127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [855ms] refreshing cache from master, [855ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.126:34221, [857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.125:44265, [858ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.124:36183, [859ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.126:34221: Network error: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221, [861ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.125:44265: Network error: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [863ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.124:36183: OK, [865ms] delaying RPC due to: Service unavailable: Master config (127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [954ms] refreshing cache from master, [954ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.126:34221, [956ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.125:44265, [957ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.124:36183, [958ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.126:34221: Network error: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221, [961ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.125:44265: Network error: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [967ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.124:36183: OK, [968ms] delaying RPC due to: Service unavailable: Master config (127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [1014ms] refreshing cache from master, [1015ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.126:34221, [1016ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.125:44265, [1018ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.124:36183, [1019ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.126:34221: Network error: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221, [1022ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.125:44265: Network error: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [1023ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.124:36183: OK, [1024ms] delaying RPC due to: Service unavailable: Master config (127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [1597ms] refreshing cache from master, [1597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.126:34221, [1599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.125:44265, [1600ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.124:36183, [1603ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.126:34221: Network error: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221, [1603ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.125:44265: Network error: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [1605ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.124:36183: OK, [1606ms] delaying RPC due to: Service unavailable: Master config (127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [3274ms] refreshing cache from master, [3275ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.126:34221, [3276ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.125:44265, [3277ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.124:36183, [3278ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.125:44265: Network error: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [3278ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.126:34221: Network error: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221, [3282ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.124:36183: OK, [3283ms] delaying RPC due to: Service unavailable: Master config (127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221, [3894ms] refreshing cache from master, [3894ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.126:34221, [3895ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.125:44265, [3896ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.124:36183, [3897ms] too many traces: truncated at 100 traces, deferred=Deferred@559077857(state=PENDING, result=null, callback=(continuation of Deferred@1471534627 after retry RPC after error@1495606074) -> (continuation of Deferred@1531322372 after retry RPC after error@1827436380) -> (continuation of Deferred@361865260 after retry RPC after error@1709014359) -> (continuation of Deferred@259074024 after retry RPC after error@11069389) -> (continuation of Deferred@404793140 after retry RPC after error@370755234) -> (continuation of Deferred@282164718 after retry RPC after error@1468032694) -> (continuation of Deferred@1241801333 after retry RPC after error@376705473) -> (continuation of Deferred@231809260 after retry RPC after error@1504727606) -> (continuation of Deferred@818840767 after retry RPC after error@109207934) -> (continuation of Deferred@38856450 after retry RPC after error@1504326812) -> (continuation of Deferred@486910058 after retry RPC after error@135901989) -> (continuation of Deferred@225721811 after retry RPC after error@91420213) -> (continuation of Deferred@889477129 after retry RPC after error@1891718423) -> (continuation of Deferred@1551332473 after retry RPC after error@1319563932) -> (continuation of Deferred@1509244753 after retry RPC after error@593932437) -> (continuation of Deferred@1154862140 after retry RPC after error@1025438760) -> (continuation of Deferred@1306610541 after retry RPC after error@1536534620) -> (continuation of Deferred@827013950 after retry RPC after error@2067674763) -> (continuation of Deferred@1011069090 after retry RPC after error@1247946814) -> (continuation of Deferred@580413575 after retry RPC after error@672700465) -> (continuation of Deferred@167766832 after retry RPC after error@1376190953) -> (continuation of Deferred@867862398 after retry RPC after error@1594837032) -> (continuation of Deferred@702516882 after retry RPC after error@1136511497), errback=(continuation of Deferred@1471534627 after retry RPC after error@1495606074) -> (continuation of Deferred@1531322372 after retry RPC after error@1827436380) -> (continuation of Deferred@361865260 after retry RPC after error@1709014359) -> (continuation of Deferred@259074024 after retry RPC after error@11069389) -> (continuation of Deferred@404793140 after retry RPC after error@370755234) -> (continuation of Deferred@282164718 after retry RPC after error@1468032694) -> (continuation of Deferred@1241801333 after retry RPC after error@376705473) -> (continuation of Deferred@231809260 after retry RPC after error@1504727606) -> (continuation of Deferred@818840767 after retry RPC after error@109207934) -> (continuation of Deferred@38856450 after retry RPC after error@1504326812) -> (continuation of Deferred@486910058 after retry RPC after error@135901989) -> (continuation of Deferred@225721811 after retry RPC after error@91420213) -> (continuation of Deferred@889477129 after retry RPC after error@1891718423) -> (continuation of Deferred@1551332473 after retry RPC after error@1319563932) -> (continuation of Deferred@1509244753 after retry RPC after error@593932437) -> (continuation of Deferred@1154862140 after retry RPC after error@1025438760) -> (continuation of Deferred@1306610541 after retry RPC after error@1536534620) -> (continuation of Deferred@827013950 after retry RPC after error@2067674763) -> (continuation of Deferred@1011069090 after retry RPC after error@1247946814) -> (continuation of Deferred@580413575 after retry RPC after error@672700465) -> (continuation of Deferred@167766832 after retry RPC after error@1376190953) -> (continuation of Deferred@867862398 after retry RPC after error@1594837032) -> (continuation of Deferred@702516882 after retry RPC after error@1136511497)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.8.25.125:44265
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=28028), Traces: [0ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.126:34221, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.125:44265, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.124:36183, [197ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.126:34221: Network error: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221, [196ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.125:44265: Network error: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [568ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.124:36183: OK, [572ms] delaying RPC due to: Service unavailable: Master config (127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [595ms] refreshing cache from master, [595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.126:34221, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.125:44265, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.124:36183, [601ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.126:34221: Network error: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221, [606ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.125:44265: Network error: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [607ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.124:36183: OK, [608ms] delaying RPC due to: Service unavailable: Master config (127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [614ms] refreshing cache from master, [614ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.126:34221, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.125:44265, [617ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.126:34221: Network error: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.124:36183, [620ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.125:44265: Network error: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [621ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.124:36183: OK, [622ms] delaying RPC due to: Service unavailable: Master config (127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [634ms] refreshing cache from master, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.126:34221, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.125:44265, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.124:36183, [638ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.126:34221: Network error: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221, [641ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.125:44265: Network error: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [644ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.124:36183: OK, [646ms] delaying RPC due to: Service unavailable: Master config (127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [674ms] refreshing cache from master, [675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.126:34221, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.125:44265, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.124:36183, [679ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.126:34221: Network error: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221, [682ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.125:44265: Network error: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [684ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.124:36183: OK, [684ms] delaying RPC due to: Service unavailable: Master config (127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [714ms] refreshing cache from master, [714ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.126:34221, [715ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.125:44265, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.124:36183, [718ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.126:34221: Network error: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221, [721ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.125:44265: Network error: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [725ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.124:36183: OK, [726ms] delaying RPC due to: Service unavailable: Master config (127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [775ms] refreshing cache from master, [775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.126:34221, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.125:44265, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.124:36183, [779ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.126:34221: Network error: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221, [782ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.125:44265: Network error: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [784ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.124:36183: OK, [784ms] delaying RPC due to: Service unavailable: Master config (127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [855ms] refreshing cache from master, [855ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.126:34221, [857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.125:44265, [858ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.124:36183, [859ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.126:34221: Network error: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221, [861ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.125:44265: Network error: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [863ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.124:36183: OK, [865ms] delaying RPC due to: Service unavailable: Master config (127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [954ms] refreshing cache from master, [954ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.126:34221, [956ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.125:44265, [957ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.124:36183, [958ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.126:34221: Network error: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221, [961ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.125:44265: Network error: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [967ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.124:36183: OK, [968ms] delaying RPC due to: Service unavailable: Master config (127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [1014ms] refreshing cache from master, [1015ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.126:34221, [1016ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.125:44265, [1018ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.124:36183, [1019ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.126:34221: Network error: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221, [1022ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.125:44265: Network error: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [1023ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.124:36183: OK, [1024ms] delaying RPC due to: Service unavailable: Master config (127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [1597ms] refreshing cache from master, [1597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.126:34221, [1599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.125:44265, [1600ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.124:36183, [1603ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.126:34221: Network error: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221, [1603ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.125:44265: Network error: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [1605ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.124:36183: OK, [1606ms] delaying RPC due to: Service unavailable: Master config (127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [3274ms] refreshing cache from master, [3275ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.126:34221, [3276ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.125:44265, [3277ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.124:36183, [3278ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.125:44265: Network error: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [3278ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.126:34221: Network error: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221, [3282ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.124:36183: OK, [3283ms] delaying RPC due to: Service unavailable: Master config (127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221, [3894ms] refreshing cache from master, [3894ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.126:34221, [3895ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.125:44265, [3896ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.124:36183, [3897ms] too many traces: truncated at 100 traces, deferred=Deferred@559077857(state=PENDING, result=null, callback=(continuation of Deferred@1471534627 after retry RPC after error@1495606074) -> (continuation of Deferred@1531322372 after retry RPC after error@1827436380) -> (continuation of Deferred@361865260 after retry RPC after error@1709014359) -> (continuation of Deferred@259074024 after retry RPC after error@11069389) -> (continuation of Deferred@404793140 after retry RPC after error@370755234) -> (continuation of Deferred@282164718 after retry RPC after error@1468032694) -> (continuation of Deferred@1241801333 after retry RPC after error@376705473) -> (continuation of Deferred@231809260 after retry RPC after error@1504727606) -> (continuation of Deferred@818840767 after retry RPC after error@109207934) -> (continuation of Deferred@38856450 after retry RPC after error@1504326812) -> (continuation of Deferred@486910058 after retry RPC after error@135901989) -> (continuation of Deferred@225721811 after retry RPC after error@91420213) -> (continuation of Deferred@889477129 after retry RPC after error@1891718423) -> (continuation of Deferred@1551332473 after retry RPC after error@1319563932) -> (continuation of Deferred@1509244753 after retry RPC after error@593932437) -> (continuation of Deferred@1154862140 after retry RPC after error@1025438760) -> (continuation of Deferred@1306610541 after retry RPC after error@1536534620) -> (continuation of Deferred@827013950 after retry RPC after error@2067674763) -> (continuation of Deferred@1011069090 after retry RPC after error@1247946814) -> (continuation of Deferred@580413575 after retry RPC after error@672700465) -> (continuation of Deferred@167766832 after retry RPC after error@1376190953) -> (continuation of Deferred@867862398 after retry RPC after error@1594837032) -> (continuation of Deferred@702516882 after retry RPC after error@1136511497), errback=(continuation of Deferred@1471534627 after retry RPC after error@1495606074) -> (continuation of Deferred@1531322372 after retry RPC after error@1827436380) -> (continuation of Deferred@361865260 after retry RPC after error@1709014359) -> (continuation of Deferred@259074024 after retry RPC after error@11069389) -> (continuation of Deferred@404793140 after retry RPC after error@370755234) -> (continuation of Deferred@282164718 after retry RPC after error@1468032694) -> (continuation of Deferred@1241801333 after retry RPC after error@376705473) -> (continuation of Deferred@231809260 after retry RPC after error@1504727606) -> (continuation of Deferred@818840767 after retry RPC after error@109207934) -> (continuation of Deferred@38856450 after retry RPC after error@1504326812) -> (continuation of Deferred@486910058 after retry RPC after error@135901989) -> (continuation of Deferred@225721811 after retry RPC after error@91420213) -> (continuation of Deferred@889477129 after retry RPC after error@1891718423) -> (continuation of Deferred@1551332473 after retry RPC after error@1319563932) -> (continuation of Deferred@1509244753 after retry RPC after error@593932437) -> (continuation of Deferred@1154862140 after retry RPC after error@1025438760) -> (continuation of Deferred@1306610541 after retry RPC after error@1536534620) -> (continuation of Deferred@827013950 after retry RPC after error@2067674763) -> (continuation of Deferred@1011069090 after retry RPC after error@1247946814) -> (continuation of Deferred@580413575 after retry RPC after error@672700465) -> (continuation of Deferred@167766832 after retry RPC after error@1376190953) -> (continuation of Deferred@867862398 after retry RPC after error@1594837032) -> (continuation of Deferred@702516882 after retry RPC after error@1136511497)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.8.25.125:44265
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
00:11:48.953 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
00:11:48.954 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=28028), Traces: [0ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.126:34221, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.125:44265, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.124:36183, [197ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.126:34221: Network error: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221, [196ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.125:44265: Network error: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [568ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.124:36183: OK, [572ms] delaying RPC due to: Service unavailable: Master config (127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [595ms] refreshing cache from master, [595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.126:34221, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.125:44265, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.124:36183, [601ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.126:34221: Network error: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221, [606ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.125:44265: Network error: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [607ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.124:36183: OK, [608ms] delaying RPC due to: Service unavailable: Master config (127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [614ms] refreshing cache from master, [614ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.126:34221, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.125:44265, [617ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.126:34221: Network error: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.124:36183, [620ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.125:44265: Network error: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [621ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.124:36183: OK, [622ms] delaying RPC due to: Service unavailable: Master config (127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [634ms] refreshing cache from master, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.126:34221, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.125:44265, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.124:36183, [638ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.126:34221: Network error: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221, [641ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.125:44265: Network error: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [644ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.124:36183: OK, [646ms] delaying RPC due to: Service unavailable: Master config (127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [674ms] refreshing cache from master, [675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.126:34221, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.125:44265, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.124:36183, [679ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.126:34221: Network error: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221, [682ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.125:44265: Network error: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [684ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.124:36183: OK, [684ms] delaying RPC due to: Service unavailable: Master config (127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [714ms] refreshing cache from master, [714ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.126:34221, [715ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.125:44265, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.124:36183, [718ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.126:34221: Network error: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221, [721ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.125:44265: Network error: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [725ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.124:36183: OK, [726ms] delaying RPC due to: Service unavailable: Master config (127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [775ms] refreshing cache from master, [775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.126:34221, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.125:44265, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.124:36183, [779ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.126:34221: Network error: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221, [782ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.125:44265: Network error: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [784ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.124:36183: OK, [784ms] delaying RPC due to: Service unavailable: Master config (127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [855ms] refreshing cache from master, [855ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.126:34221, [857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.125:44265, [858ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.124:36183, [859ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.126:34221: Network error: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221, [861ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.125:44265: Network error: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [863ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.124:36183: OK, [865ms] delaying RPC due to: Service unavailable: Master config (127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [954ms] refreshing cache from master, [954ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.126:34221, [956ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.125:44265, [957ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.124:36183, [958ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.126:34221: Network error: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221, [961ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.125:44265: Network error: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [967ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.124:36183: OK, [968ms] delaying RPC due to: Service unavailable: Master config (127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [1014ms] refreshing cache from master, [1015ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.126:34221, [1016ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.125:44265, [1018ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.124:36183, [1019ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.126:34221: Network error: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221, [1022ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.125:44265: Network error: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [1023ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.124:36183: OK, [1024ms] delaying RPC due to: Service unavailable: Master config (127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [1597ms] refreshing cache from master, [1597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.126:34221, [1599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.125:44265, [1600ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.124:36183, [1603ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.126:34221: Network error: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221, [1603ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.125:44265: Network error: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [1605ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.124:36183: OK, [1606ms] delaying RPC due to: Service unavailable: Master config (127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [3274ms] refreshing cache from master, [3275ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.126:34221, [3276ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.125:44265, [3277ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.124:36183, [3278ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.125:44265: Network error: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [3278ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.126:34221: Network error: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221, [3282ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.124:36183: OK, [3283ms] delaying RPC due to: Service unavailable: Master config (127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221, [3894ms] refreshing cache from master, [3894ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.126:34221, [3895ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.125:44265, [3896ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.124:36183, [3897ms] too many traces: truncated at 100 traces, deferred=Deferred@559077857(state=PENDING, result=null, callback=(continuation of Deferred@1471534627 after retry RPC after error@1495606074) -> (continuation of Deferred@1531322372 after retry RPC after error@1827436380) -> (continuation of Deferred@361865260 after retry RPC after error@1709014359) -> (continuation of Deferred@259074024 after retry RPC after error@11069389) -> (continuation of Deferred@404793140 after retry RPC after error@370755234) -> (continuation of Deferred@282164718 after retry RPC after error@1468032694) -> (continuation of Deferred@1241801333 after retry RPC after error@376705473) -> (continuation of Deferred@231809260 after retry RPC after error@1504727606) -> (continuation of Deferred@818840767 after retry RPC after error@109207934) -> (continuation of Deferred@38856450 after retry RPC after error@1504326812) -> (continuation of Deferred@486910058 after retry RPC after error@135901989) -> (continuation of Deferred@225721811 after retry RPC after error@91420213) -> (continuation of Deferred@889477129 after retry RPC after error@1891718423) -> (continuation of Deferred@1551332473 after retry RPC after error@1319563932) -> (continuation of Deferred@1509244753 after retry RPC after error@593932437) -> (continuation of Deferred@1154862140 after retry RPC after error@1025438760) -> (continuation of Deferred@1306610541 after retry RPC after error@1536534620) -> (continuation of Deferred@827013950 after retry RPC after error@2067674763) -> (continuation of Deferred@1011069090 after retry RPC after error@1247946814) -> (continuation of Deferred@580413575 after retry RPC after error@672700465) -> (continuation of Deferred@167766832 after retry RPC after error@1376190953) -> (continuation of Deferred@867862398 after retry RPC after error@1594837032) -> (continuation of Deferred@702516882 after retry RPC after error@1136511497), errback=(continuation of Deferred@1471534627 after retry RPC after error@1495606074) -> (continuation of Deferred@1531322372 after retry RPC after error@1827436380) -> (continuation of Deferred@361865260 after retry RPC after error@1709014359) -> (continuation of Deferred@259074024 after retry RPC after error@11069389) -> (continuation of Deferred@404793140 after retry RPC after error@370755234) -> (continuation of Deferred@282164718 after retry RPC after error@1468032694) -> (continuation of Deferred@1241801333 after retry RPC after error@376705473) -> (continuation of Deferred@231809260 after retry RPC after error@1504727606) -> (continuation of Deferred@818840767 after retry RPC after error@109207934) -> (continuation of Deferred@38856450 after retry RPC after error@1504326812) -> (continuation of Deferred@486910058 after retry RPC after error@135901989) -> (continuation of Deferred@225721811 after retry RPC after error@91420213) -> (continuation of Deferred@889477129 after retry RPC after error@1891718423) -> (continuation of Deferred@1551332473 after retry RPC after error@1319563932) -> (continuation of Deferred@1509244753 after retry RPC after error@593932437) -> (continuation of Deferred@1154862140 after retry RPC after error@1025438760) -> (continuation of Deferred@1306610541 after retry RPC after error@1536534620) -> (continuation of Deferred@827013950 after retry RPC after error@2067674763) -> (continuation of Deferred@1011069090 after retry RPC after error@1247946814) -> (continuation of Deferred@580413575 after retry RPC after error@672700465) -> (continuation of Deferred@167766832 after retry RPC after error@1376190953) -> (continuation of Deferred@867862398 after retry RPC after error@1594837032) -> (continuation of Deferred@702516882 after retry RPC after error@1136511497)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.8.25.125:44265
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=28028), Traces: [0ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.126:34221, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.125:44265, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.124:36183, [197ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.126:34221: Network error: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221, [196ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.125:44265: Network error: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [568ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.124:36183: OK, [572ms] delaying RPC due to: Service unavailable: Master config (127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [595ms] refreshing cache from master, [595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.126:34221, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.125:44265, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.124:36183, [601ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.126:34221: Network error: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221, [606ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.125:44265: Network error: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [607ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.124:36183: OK, [608ms] delaying RPC due to: Service unavailable: Master config (127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [614ms] refreshing cache from master, [614ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.126:34221, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.125:44265, [617ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.126:34221: Network error: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.124:36183, [620ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.125:44265: Network error: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [621ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.124:36183: OK, [622ms] delaying RPC due to: Service unavailable: Master config (127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [634ms] refreshing cache from master, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.126:34221, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.125:44265, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.124:36183, [638ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.126:34221: Network error: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221, [641ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.125:44265: Network error: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [644ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.124:36183: OK, [646ms] delaying RPC due to: Service unavailable: Master config (127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [674ms] refreshing cache from master, [675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.126:34221, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.125:44265, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.124:36183, [679ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.126:34221: Network error: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221, [682ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.125:44265: Network error: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [684ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.124:36183: OK, [684ms] delaying RPC due to: Service unavailable: Master config (127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [714ms] refreshing cache from master, [714ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.126:34221, [715ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.125:44265, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.124:36183, [718ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.126:34221: Network error: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221, [721ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.125:44265: Network error: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [725ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.124:36183: OK, [726ms] delaying RPC due to: Service unavailable: Master config (127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [775ms] refreshing cache from master, [775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.126:34221, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.125:44265, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.124:36183, [779ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.126:34221: Network error: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221, [782ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.125:44265: Network error: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [784ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.124:36183: OK, [784ms] delaying RPC due to: Service unavailable: Master config (127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [855ms] refreshing cache from master, [855ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.126:34221, [857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.125:44265, [858ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.124:36183, [859ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.126:34221: Network error: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221, [861ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.125:44265: Network error: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [863ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.124:36183: OK, [865ms] delaying RPC due to: Service unavailable: Master config (127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [954ms] refreshing cache from master, [954ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.126:34221, [956ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.125:44265, [957ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.124:36183, [958ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.126:34221: Network error: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221, [961ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.125:44265: Network error: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [967ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.124:36183: OK, [968ms] delaying RPC due to: Service unavailable: Master config (127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [1014ms] refreshing cache from master, [1015ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.126:34221, [1016ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.125:44265, [1018ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.124:36183, [1019ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.126:34221: Network error: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221, [1022ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.125:44265: Network error: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [1023ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.124:36183: OK, [1024ms] delaying RPC due to: Service unavailable: Master config (127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [1597ms] refreshing cache from master, [1597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.126:34221, [1599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.125:44265, [1600ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.124:36183, [1603ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.126:34221: Network error: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221, [1603ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.125:44265: Network error: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [1605ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.124:36183: OK, [1606ms] delaying RPC due to: Service unavailable: Master config (127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [3274ms] refreshing cache from master, [3275ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.126:34221, [3276ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.125:44265, [3277ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.124:36183, [3278ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.125:44265: Network error: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265, [3278ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.126:34221: Network error: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221, [3282ms] Sub RPC ConnectToMaster: received response from server master-127.8.25.124:36183: OK, [3283ms] delaying RPC due to: Service unavailable: Master config (127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221, [3894ms] refreshing cache from master, [3894ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.126:34221, [3895ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.125:44265, [3896ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.25.124:36183, [3897ms] too many traces: truncated at 100 traces, deferred=Deferred@559077857(state=PENDING, result=null, callback=(continuation of Deferred@1471534627 after retry RPC after error@1495606074) -> (continuation of Deferred@1531322372 after retry RPC after error@1827436380) -> (continuation of Deferred@361865260 after retry RPC after error@1709014359) -> (continuation of Deferred@259074024 after retry RPC after error@11069389) -> (continuation of Deferred@404793140 after retry RPC after error@370755234) -> (continuation of Deferred@282164718 after retry RPC after error@1468032694) -> (continuation of Deferred@1241801333 after retry RPC after error@376705473) -> (continuation of Deferred@231809260 after retry RPC after error@1504727606) -> (continuation of Deferred@818840767 after retry RPC after error@109207934) -> (continuation of Deferred@38856450 after retry RPC after error@1504326812) -> (continuation of Deferred@486910058 after retry RPC after error@135901989) -> (continuation of Deferred@225721811 after retry RPC after error@91420213) -> (continuation of Deferred@889477129 after retry RPC after error@1891718423) -> (continuation of Deferred@1551332473 after retry RPC after error@1319563932) -> (continuation of Deferred@1509244753 after retry RPC after error@593932437) -> (continuation of Deferred@1154862140 after retry RPC after error@1025438760) -> (continuation of Deferred@1306610541 after retry RPC after error@1536534620) -> (continuation of Deferred@827013950 after retry RPC after error@2067674763) -> (continuation of Deferred@1011069090 after retry RPC after error@1247946814) -> (continuation of Deferred@580413575 after retry RPC after error@672700465) -> (continuation of Deferred@167766832 after retry RPC after error@1376190953) -> (continuation of Deferred@867862398 after retry RPC after error@1594837032) -> (continuation of Deferred@702516882 after retry RPC after error@1136511497), errback=(continuation of Deferred@1471534627 after retry RPC after error@1495606074) -> (continuation of Deferred@1531322372 after retry RPC after error@1827436380) -> (continuation of Deferred@361865260 after retry RPC after error@1709014359) -> (continuation of Deferred@259074024 after retry RPC after error@11069389) -> (continuation of Deferred@404793140 after retry RPC after error@370755234) -> (continuation of Deferred@282164718 after retry RPC after error@1468032694) -> (continuation of Deferred@1241801333 after retry RPC after error@376705473) -> (continuation of Deferred@231809260 after retry RPC after error@1504727606) -> (continuation of Deferred@818840767 after retry RPC after error@109207934) -> (continuation of Deferred@38856450 after retry RPC after error@1504326812) -> (continuation of Deferred@486910058 after retry RPC after error@135901989) -> (continuation of Deferred@225721811 after retry RPC after error@91420213) -> (continuation of Deferred@889477129 after retry RPC after error@1891718423) -> (continuation of Deferred@1551332473 after retry RPC after error@1319563932) -> (continuation of Deferred@1509244753 after retry RPC after error@593932437) -> (continuation of Deferred@1154862140 after retry RPC after error@1025438760) -> (continuation of Deferred@1306610541 after retry RPC after error@1536534620) -> (continuation of Deferred@827013950 after retry RPC after error@2067674763) -> (continuation of Deferred@1011069090 after retry RPC after error@1247946814) -> (continuation of Deferred@580413575 after retry RPC after error@672700465) -> (continuation of Deferred@167766832 after retry RPC after error@1376190953) -> (continuation of Deferred@867862398 after retry RPC after error@1594837032) -> (continuation of Deferred@702516882 after retry RPC after error@1136511497)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.8.25.126:34221,127.8.25.125:44265,127.8.25.124:36183) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.126:34221(127.8.25.126:34221): Connection refused: /127.8.25.126:34221,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.25.125:44265(127.8.25.125:44265): Connection refused: /127.8.25.125:44265
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.8.25.125:44265
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)