16:16:21.363 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
16:16:21.376 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskkW_qFj/test-tmp
16:16:21.378 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
16:16:21.382 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskkW_qFj/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
16:16:21.627 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
16:16:21.776 [DEBUG - main] (MiniKuduCluster.java:195) Response:
16:16:21.777 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
16:16:21.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:21.781132 24360 mini_oidc.cc:152] Starting JWKS server
16:16:21.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:21.785019 24360 webserver.cc:460] Webserver started at http://127.0.0.1:38933/ using document root <none> and password file <none>
16:16:21.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:21.785606 24360 mini_oidc.cc:189] Starting OIDC Discovery server
16:16:21.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:21.787686 24360 webserver.cc:460] Webserver started at http://127.0.0.1:37587/ using document root <none> and password file <none>
16:16:21.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-26T16:16:21Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
16:16:21.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-26T16:16:21Z Disabled control of system clock
16:16:21.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:21.839857 24360 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskkW_qFj/build/tsan/bin/kudu
16:16:21.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskkW_qFj/build/tsan/bin/kudu
16:16:21.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/master-0/wal
16:16:21.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/master-0/data
16:16:21.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:16:21.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:16:21.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:16:21.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:16:21.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:16:21.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:16:21.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/master-0/logs
16:16:21.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/master-0/data/info.pb
16:16:21.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:16:21.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:16:21.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:16:21.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:16:21.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:16:21.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:16:21.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:16:21.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:16:21.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:16:21.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:16:21.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:16:21.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:16:21.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.202.62:40007
16:16:21.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.202.62
16:16:21.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:16:21.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.202.20:43197
16:16:21.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:16:21.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:16:21.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:16:21.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:16:21.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.202.62:40007,127.23.202.61:36223,127.23.202.60:32791
16:16:21.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:16:21.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38933/jwks.json with env {}
16:16:22.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:22.148149 24372 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:16:22.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:22.148716 24372 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:16:22.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:22.149349 24372 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:16:22.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:22.189231 24372 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:16:22.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:22.189530 24372 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:16:22.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:22.189724 24372 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:16:22.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:22.189952 24372 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:16:22.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:22.190215 24372 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:16:22.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:22.190392 24372 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38933/jwks.json
16:16:22.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:22.226486 24372 master_runner.cc:386] Master server non-default flags:
16:16:22.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:16:22.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.202.20:43197
16:16:22.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:16:22.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:16:22.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/master-0/data
16:16:22.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/master-0/wal
16:16:22.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:16:22.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.202.62:40007,127.23.202.61:36223,127.23.202.60:32791
16:16:22.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:16:22.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:16:22.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:16:22.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.202.62:40007
16:16:22.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:16:22.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:16:22.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:16:22.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38933/jwks.json
16:16:22.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:16:22.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:16:22.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/master-0/data/info.pb
16:16:22.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.202.62
16:16:22.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:16:22.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:16:22.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:16:22.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:16:22.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:16:22.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:16:22.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/master-0/logs
16:16:22.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:16:22.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:16:22.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:16:22.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:16:22.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
16:16:22.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
16:16:22.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:16:22.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 14:44:58 UTC on ccycloud.quasar-hxpbms.root.comops.site
16:16:22.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4674114
16:16:22.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:16:22.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:22.228065 24372 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:16:22.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:22.229995 24372 file_cache.cc:493] Constructed file cache file cache with capacity 419430
16:16:22.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:22.248533 24379 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:16:23.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:22.248821 24378 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:16:23.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:22.250875 24372 server_base.cc:1034] running on GCE node
16:16:23.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:22.249482 24381 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:16:23.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:23.445427 24372 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:16:23.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:23.448987 24372 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:16:23.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:23.450560 24372 hybrid_clock.cc:648] HybridClock initialized: now 1790439383450511 us; error 69 us; skew 500 ppm
16:16:23.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:23.451601 24372 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:16:23.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:23.459108 24372 webserver.cc:460] Webserver started at http://127.23.202.62:41375/ using document root <none> and password file <none>
16:16:23.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:23.460132 24372 fs_manager.cc:359] Metadata directory not provided
16:16:23.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:23.460323 24372 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:16:23.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:23.460717 24372 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
16:16:23.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:23.465674 24372 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/master-0/data/instance:
16:16:23.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9b35ffacf87b4bbfb86b20c3359d87f2"
16:16:23.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:16:23 on dist-test-slave-fnz7"
16:16:23.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:23.466910 24372 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/master-0/wal/instance:
16:16:23.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9b35ffacf87b4bbfb86b20c3359d87f2"
16:16:23.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:16:23 on dist-test-slave-fnz7"
16:16:23.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:23.474411 24372 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.000s
16:16:23.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:23.480270 24388 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:16:23.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:23.481480 24372 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.003s
16:16:23.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:23.481786 24372 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/master-0/data,/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/master-0/wal
16:16:23.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9b35ffacf87b4bbfb86b20c3359d87f2"
16:16:23.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:16:23 on dist-test-slave-fnz7"
16:16:23.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:23.482136 24372 fs_report.cc:383] FS layout report
16:16:23.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:16:23.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/master-0/wal
16:16:23.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/master-0/wal
16:16:23.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/master-0/data/data
16:16:23.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:16:23.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:16:23.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:16:23.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:16:23.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:16:23.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:16:23.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:16:23.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:16:23.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:16:23.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:16:23.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:16:23.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:23.560298 24372 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
16:16:23.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:23.561782 24372 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:16:23.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:23.562201 24372 kserver.cc:163] Server-wide thread pool size limit: 3276
16:16:23.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:23.632885 24372 rpc_server.cc:307] RPC server started. Bound to: 127.23.202.62:40007
16:16:23.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:23.632978 24439 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.23.202.62:40007 every 8 connection(s)
16:16:23.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:23.635732 24372 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/master-0/data/info.pb
16:16:23.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:23.640549 24360 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskkW_qFj/build/tsan/bin/kudu as pid 24372
16:16:23.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:23.641103 24360 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/master-0/wal/instance
16:16:23.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:23.641644 24440 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
16:16:23.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:23.646339 24360 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskkW_qFj/build/tsan/bin/kudu
16:16:23.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskkW_qFj/build/tsan/bin/kudu
16:16:23.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/master-1/wal
16:16:23.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/master-1/data
16:16:23.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:16:23.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:16:23.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:16:23.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:16:23.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:16:23.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:16:23.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/master-1/logs
16:16:23.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/master-1/data/info.pb
16:16:23.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:16:23.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:16:23.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:16:23.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:16:23.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:16:23.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:16:23.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:16:23.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:16:23.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:16:23.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:16:23.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:16:23.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:16:23.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.202.61:36223
16:16:23.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.202.61
16:16:23.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:16:23.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.202.20:43197
16:16:23.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:16:23.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:16:23.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:16:23.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:16:23.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.202.62:40007,127.23.202.61:36223,127.23.202.60:32791
16:16:23.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:16:23.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38933/jwks.json with env {}
16:16:23.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:23.656900 24440 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.202.62" port: 40007 } has no permanent_uuid. Determining permanent_uuid...
16:16:23.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:23.689471 24440 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.202.61" port: 36223 } has no permanent_uuid. Determining permanent_uuid...
16:16:23.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:23.691920 24390 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.202.61:36223: connect: Connection refused (error 111)
16:16:23.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:23.696358 24440 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.202.61:36223: Network error: Client connection negotiation failed: client connection to 127.23.202.61:36223: connect: Connection refused (error 111)
16:16:23.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:23.745926 24440 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.202.61" port: 36223 } attempt: 1
16:16:23.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:23.750844 24440 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.202.61:36223: Network error: Client connection negotiation failed: client connection to 127.23.202.61:36223: connect: Connection refused (error 111)
16:16:23.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:23.819378 24440 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.202.61" port: 36223 } attempt: 2
16:16:23.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:23.823640 24440 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.202.61:36223: Network error: Client connection negotiation failed: client connection to 127.23.202.61:36223: connect: Connection refused (error 111)
16:16:23.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:23.915093 24440 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.202.61" port: 36223 } attempt: 3
16:16:23.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:23.919261 24440 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.202.61:36223: Network error: Client connection negotiation failed: client connection to 127.23.202.61:36223: connect: Connection refused (error 111)
16:16:23.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:23.961695 24442 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:16:23.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:23.962281 24442 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:16:23.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:23.962836 24442 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:16:23.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:23.993732 24442 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:16:23.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:23.994014 24442 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:16:23.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:23.994202 24442 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:16:23.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:23.994390 24442 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:16:23.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:23.994565 24442 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:16:23.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:23.994694 24442 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38933/jwks.json
16:16:24.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:24.031193 24442 master_runner.cc:386] Master server non-default flags:
16:16:24.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:16:24.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.202.20:43197
16:16:24.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:16:24.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:16:24.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/master-1/data
16:16:24.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/master-1/wal
16:16:24.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:16:24.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.202.62:40007,127.23.202.61:36223,127.23.202.60:32791
16:16:24.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:16:24.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:16:24.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:16:24.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.202.61:36223
16:16:24.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:16:24.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:16:24.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:16:24.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38933/jwks.json
16:16:24.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:16:24.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:16:24.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/master-1/data/info.pb
16:16:24.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.202.61
16:16:24.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:16:24.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:16:24.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:16:24.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:16:24.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:16:24.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:16:24.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/master-1/logs
16:16:24.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:16:24.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:16:24.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:16:24.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:16:24.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
16:16:24.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
16:16:24.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:16:24.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 14:44:58 UTC on ccycloud.quasar-hxpbms.root.comops.site
16:16:24.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4674114
16:16:24.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:16:24.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:24.032457 24442 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:16:24.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:24.034516 24442 file_cache.cc:493] Constructed file cache file cache with capacity 419430
16:16:24.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:24.052762 24451 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:16:24.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:24.062829 24440 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.202.61" port: 36223 } attempt: 4
16:16:24.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:24.066620 24440 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.202.61:36223: Network error: Client connection negotiation failed: client connection to 127.23.202.61:36223: connect: Connection refused (error 111)
16:16:24.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:24.366156 24440 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.202.61" port: 36223 } attempt: 5
16:16:24.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:24.370500 24440 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.202.61:36223: Network error: Client connection negotiation failed: client connection to 127.23.202.61:36223: connect: Connection refused (error 111)
16:16:24.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:24.918210 24440 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.202.61" port: 36223 } attempt: 6
16:16:24.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:24.925926 24440 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.202.61:36223: Network error: Client connection negotiation failed: client connection to 127.23.202.61:36223: connect: Connection refused (error 111)
16:16:25.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:24.052791 24452 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:16:25.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:24.053547 24454 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:16:25.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:24.054895 24442 server_base.cc:1034] running on GCE node
16:16:25.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:25.252768 24442 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:16:25.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:25.256155 24442 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:16:25.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:25.257702 24442 hybrid_clock.cc:648] HybridClock initialized: now 1790439385257662 us; error 57 us; skew 500 ppm
16:16:25.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:25.258981 24442 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:16:25.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:25.266316 24442 webserver.cc:460] Webserver started at http://127.23.202.61:35971/ using document root <none> and password file <none>
16:16:25.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:25.267398 24442 fs_manager.cc:359] Metadata directory not provided
16:16:25.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:25.267637 24442 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:16:25.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:25.268033 24442 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
16:16:25.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:25.273049 24442 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/master-1/data/instance:
16:16:25.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2a097e36445f4132acfe45358f72d2b9"
16:16:25.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:16:25 on dist-test-slave-fnz7"
16:16:25.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:25.274246 24442 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/master-1/wal/instance:
16:16:25.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2a097e36445f4132acfe45358f72d2b9"
16:16:25.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:16:25 on dist-test-slave-fnz7"
16:16:25.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:25.282086 24442 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.009s sys 0.001s
16:16:25.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:25.288110 24463 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:16:25.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:25.289347 24442 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.000s sys 0.004s
16:16:25.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:25.289690 24442 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/master-1/data,/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/master-1/wal
16:16:25.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2a097e36445f4132acfe45358f72d2b9"
16:16:25.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:16:25 on dist-test-slave-fnz7"
16:16:25.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:25.290006 24442 fs_report.cc:383] FS layout report
16:16:25.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:16:25.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/master-1/wal
16:16:25.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/master-1/wal
16:16:25.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/master-1/data/data
16:16:25.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:16:25.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:16:25.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:16:25.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:16:25.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:16:25.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:16:25.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:16:25.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:16:25.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:16:25.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:16:25.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:16:25.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:25.346771 24442 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
16:16:25.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:25.348330 24442 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:16:25.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:25.348750 24442 kserver.cc:163] Server-wide thread pool size limit: 3276
16:16:25.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:25.426319 24442 rpc_server.cc:307] RPC server started. Bound to: 127.23.202.61:36223
16:16:25.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:25.426388 24514 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.23.202.61:36223 every 8 connection(s)
16:16:25.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:25.429193 24442 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/master-1/data/info.pb
16:16:25.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:25.434922 24515 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
16:16:25.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:25.436285 24360 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskkW_qFj/build/tsan/bin/kudu as pid 24442
16:16:25.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:25.436614 24360 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/master-1/wal/instance
16:16:25.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:25.438665 24360 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskkW_qFj/build/tsan/bin/kudu
16:16:25.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskkW_qFj/build/tsan/bin/kudu
16:16:25.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/master-2/wal
16:16:25.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/master-2/data
16:16:25.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:16:25.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:16:25.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:16:25.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:16:25.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:16:25.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:16:25.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/master-2/logs
16:16:25.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/master-2/data/info.pb
16:16:25.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:16:25.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:16:25.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:16:25.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:16:25.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:16:25.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:16:25.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:16:25.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:16:25.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:16:25.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:16:25.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:16:25.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:16:25.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.202.60:32791
16:16:25.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.202.60
16:16:25.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:16:25.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.202.20:43197
16:16:25.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:16:25.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:16:25.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:16:25.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:16:25.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.202.62:40007,127.23.202.61:36223,127.23.202.60:32791
16:16:25.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:16:25.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38933/jwks.json with env {}
16:16:25.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:25.451725 24515 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.202.62" port: 40007 } has no permanent_uuid. Determining permanent_uuid...
16:16:25.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:25.484182 24515 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.202.61" port: 36223 } has no permanent_uuid. Determining permanent_uuid...
16:16:25.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:25.504228 24515 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.202.60" port: 32791 } has no permanent_uuid. Determining permanent_uuid...
16:16:25.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:25.506601 24465 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.202.60:32791: connect: Connection refused (error 111)
16:16:25.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:25.512132 24515 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.202.60:32791: Network error: Client connection negotiation failed: client connection to 127.23.202.60:32791: connect: Connection refused (error 111)
16:16:25.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:25.561676 24515 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.202.60" port: 32791 } attempt: 1
16:16:25.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:25.565872 24515 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.202.60:32791: Network error: Client connection negotiation failed: client connection to 127.23.202.60:32791: connect: Connection refused (error 111)
16:16:25.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:25.634374 24515 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.202.60" port: 32791 } attempt: 2
16:16:25.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:25.639118 24515 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.202.60:32791: Network error: Client connection negotiation failed: client connection to 127.23.202.60:32791: connect: Connection refused (error 111)
16:16:25.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:25.730677 24515 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.202.60" port: 32791 } attempt: 3
16:16:25.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:25.734481 24515 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.202.60:32791: Network error: Client connection negotiation failed: client connection to 127.23.202.60:32791: connect: Connection refused (error 111)
16:16:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:25.763600 24517 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:16:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:25.764176 24517 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:16:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:25.764633 24517 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:16:25.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:25.795969 24517 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:16:25.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:25.796270 24517 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:16:25.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:25.796490 24517 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:16:25.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:25.796669 24517 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:16:25.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:25.796904 24517 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:16:25.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:25.797108 24517 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38933/jwks.json
16:16:25.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:25.832439 24517 master_runner.cc:386] Master server non-default flags:
16:16:25.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:16:25.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.202.20:43197
16:16:25.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:16:25.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:16:25.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/master-2/data
16:16:25.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/master-2/wal
16:16:25.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:16:25.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.202.62:40007,127.23.202.61:36223,127.23.202.60:32791
16:16:25.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:16:25.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:16:25.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:16:25.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.202.60:32791
16:16:25.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:16:25.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:16:25.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:16:25.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38933/jwks.json
16:16:25.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:16:25.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:16:25.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/master-2/data/info.pb
16:16:25.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.202.60
16:16:25.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:16:25.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:16:25.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:16:25.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:16:25.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:16:25.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:16:25.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/master-2/logs
16:16:25.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:16:25.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:16:25.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:16:25.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:16:25.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
16:16:25.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
16:16:25.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:16:25.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 14:44:58 UTC on ccycloud.quasar-hxpbms.root.comops.site
16:16:25.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4674114
16:16:25.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:16:25.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:25.833624 24517 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:16:25.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:25.835614 24517 file_cache.cc:493] Constructed file cache file cache with capacity 419430
16:16:25.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:25.855510 24530 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:16:25.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:25.877938 24515 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.202.60" port: 32791 } attempt: 4
16:16:25.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:25.882567 24515 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.202.60:32791: Network error: Client connection negotiation failed: client connection to 127.23.202.60:32791: connect: Connection refused (error 111)
16:16:25.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:25.986660 24440 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.202.61" port: 36223 } attempt: 7
16:16:26.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:26.006194 24440 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.202.60" port: 32791 } has no permanent_uuid. Determining permanent_uuid...
16:16:26.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:26.013341 24440 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.202.60:32791: Network error: Client connection negotiation failed: client connection to 127.23.202.60:32791: connect: Connection refused (error 111)
16:16:26.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:26.071973 24440 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.202.60" port: 32791 } attempt: 1
16:16:26.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:26.076517 24440 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.202.60:32791: Network error: Client connection negotiation failed: client connection to 127.23.202.60:32791: connect: Connection refused (error 111)
16:16:26.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:26.158248 24440 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.202.60" port: 32791 } attempt: 2
16:16:26.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:26.162912 24440 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.202.60:32791: Network error: Client connection negotiation failed: client connection to 127.23.202.60:32791: connect: Connection refused (error 111)
16:16:26.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:26.182031 24515 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.202.60" port: 32791 } attempt: 5
16:16:26.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:26.186856 24515 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.202.60:32791: Network error: Client connection negotiation failed: client connection to 127.23.202.60:32791: connect: Connection refused (error 111)
16:16:26.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:26.248569 24440 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.202.60" port: 32791 } attempt: 3
16:16:26.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:26.252677 24440 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.202.60:32791: Network error: Client connection negotiation failed: client connection to 127.23.202.60:32791: connect: Connection refused (error 111)
16:16:26.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:26.393383 24440 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.202.60" port: 32791 } attempt: 4
16:16:26.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:26.398627 24440 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.202.60:32791: Network error: Client connection negotiation failed: client connection to 127.23.202.60:32791: connect: Connection refused (error 111)
16:16:26.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:26.682102 24440 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.202.60" port: 32791 } attempt: 5
16:16:26.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:26.686553 24440 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.202.60:32791: Network error: Client connection negotiation failed: client connection to 127.23.202.60:32791: connect: Connection refused (error 111)
16:16:26.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:26.734283 24515 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.202.60" port: 32791 } attempt: 6
16:16:26.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:26.741285 24515 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.202.60:32791: Network error: Client connection negotiation failed: client connection to 127.23.202.60:32791: connect: Connection refused (error 111)
16:16:27.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:25.855516 24527 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:16:27.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:25.855569 24528 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:16:27.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:25.857381 24517 server_base.cc:1034] running on GCE node
16:16:27.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.103642 24517 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:16:27.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.107002 24517 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:16:27.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.108502 24517 hybrid_clock.cc:648] HybridClock initialized: now 1790439387108437 us; error 85 us; skew 500 ppm
16:16:27.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.109378 24517 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:16:27.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.116386 24517 webserver.cc:460] Webserver started at http://127.23.202.60:41551/ using document root <none> and password file <none>
16:16:27.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.117316 24517 fs_manager.cc:359] Metadata directory not provided
16:16:27.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.117504 24517 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:16:27.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.117883 24517 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
16:16:27.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.125006 24517 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/master-2/data/instance:
16:16:27.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "96e6a3a6c134457dab45358ef07a08e3"
16:16:27.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:16:27 on dist-test-slave-fnz7"
16:16:27.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.126652 24517 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/master-2/wal/instance:
16:16:27.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "96e6a3a6c134457dab45358ef07a08e3"
16:16:27.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:16:27 on dist-test-slave-fnz7"
16:16:27.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.136955 24517 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.004s sys 0.004s
16:16:27.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.143869 24541 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:16:27.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.145071 24517 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
16:16:27.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.145484 24517 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/master-2/data,/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/master-2/wal
16:16:27.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "96e6a3a6c134457dab45358ef07a08e3"
16:16:27.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:16:27 on dist-test-slave-fnz7"
16:16:27.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.145893 24517 fs_report.cc:383] FS layout report
16:16:27.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:16:27.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/master-2/wal
16:16:27.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/master-2/wal
16:16:27.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/master-2/data/data
16:16:27.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:16:27.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:16:27.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:16:27.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:16:27.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:16:27.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:16:27.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:16:27.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:16:27.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:16:27.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:16:27.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:16:27.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.193832 24517 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
16:16:27.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.195648 24517 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:16:27.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.196211 24517 kserver.cc:163] Server-wide thread pool size limit: 3276
16:16:27.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.239015 24440 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.202.60" port: 32791 } attempt: 6
16:16:27.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:27.246847 24440 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.202.60:32791: Network error: Client connection negotiation failed: client connection to 127.23.202.60:32791: connect: Connection refused (error 111)
16:16:27.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.275027 24517 rpc_server.cc:307] RPC server started. Bound to: 127.23.202.60:32791
16:16:27.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.275130 24594 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.23.202.60:32791 every 8 connection(s)
16:16:27.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.278683 24517 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/master-2/data/info.pb
16:16:27.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.279454 24360 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskkW_qFj/build/tsan/bin/kudu as pid 24517
16:16:27.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.279896 24360 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/master-2/wal/instance
16:16:27.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.288898 24595 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
16:16:27.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.306517 24595 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.202.62" port: 40007 } has no permanent_uuid. Determining permanent_uuid...
16:16:27.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.334233 24595 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.202.61" port: 36223 } has no permanent_uuid. Determining permanent_uuid...
16:16:27.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.346520 24595 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.202.60" port: 32791 } has no permanent_uuid. Determining permanent_uuid...
16:16:27.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.364015 24595 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 96e6a3a6c134457dab45358ef07a08e3: Bootstrap starting.
16:16:27.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.370673 24595 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 96e6a3a6c134457dab45358ef07a08e3: Neither blocks nor log segments found. Creating new log.
16:16:27.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.372453 24595 log.cc:824] T 00000000000000000000000000000000 P 96e6a3a6c134457dab45358ef07a08e3: Log is configured to *not* fsync() on all Append() calls
16:16:27.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.377382 24595 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 96e6a3a6c134457dab45358ef07a08e3: No bootstrap required, opened a new log
16:16:27.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.399786 24595 raft_consensus.cc:348] T 00000000000000000000000000000000 P 96e6a3a6c134457dab45358ef07a08e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9b35ffacf87b4bbfb86b20c3359d87f2" member_type: VOTER last_known_addr { host: "127.23.202.62" port: 40007 } } peers { permanent_uuid: "2a097e36445f4132acfe45358f72d2b9" member_type: VOTER last_known_addr { host: "127.23.202.61" port: 36223 } } peers { permanent_uuid: "96e6a3a6c134457dab45358ef07a08e3" member_type: VOTER last_known_addr { host: "127.23.202.60" port: 32791 } }
16:16:27.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.400975 24595 raft_consensus.cc:374] T 00000000000000000000000000000000 P 96e6a3a6c134457dab45358ef07a08e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
16:16:27.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.401506 24595 raft_consensus.cc:729] T 00000000000000000000000000000000 P 96e6a3a6c134457dab45358ef07a08e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 96e6a3a6c134457dab45358ef07a08e3, State: Initialized, Role: FOLLOWER
16:16:27.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.402496 24595 consensus_queue.cc:260] T 00000000000000000000000000000000 P 96e6a3a6c134457dab45358ef07a08e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9b35ffacf87b4bbfb86b20c3359d87f2" member_type: VOTER last_known_addr { host: "127.23.202.62" port: 40007 } } peers { permanent_uuid: "2a097e36445f4132acfe45358f72d2b9" member_type: VOTER last_known_addr { host: "127.23.202.61" port: 36223 } } peers { permanent_uuid: "96e6a3a6c134457dab45358ef07a08e3" member_type: VOTER last_known_addr { host: "127.23.202.60" port: 32791 } }
16:16:27.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.404970 24604 sys_catalog.cc:455] T 00000000000000000000000000000000 P 96e6a3a6c134457dab45358ef07a08e3 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9b35ffacf87b4bbfb86b20c3359d87f2" member_type: VOTER last_known_addr { host: "127.23.202.62" port: 40007 } } peers { permanent_uuid: "2a097e36445f4132acfe45358f72d2b9" member_type: VOTER last_known_addr { host: "127.23.202.61" port: 36223 } } peers { permanent_uuid: "96e6a3a6c134457dab45358ef07a08e3" member_type: VOTER last_known_addr { host: "127.23.202.60" port: 32791 } } }
16:16:27.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.405646 24604 sys_catalog.cc:458] T 00000000000000000000000000000000 P 96e6a3a6c134457dab45358ef07a08e3 [sys.catalog]: This master's current role is: FOLLOWER
16:16:27.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.406309 24595 sys_catalog.cc:564] T 00000000000000000000000000000000 P 96e6a3a6c134457dab45358ef07a08e3 [sys.catalog]: configured and running, proceeding with master startup.
16:16:27.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:27.438879 24615 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 96e6a3a6c134457dab45358ef07a08e3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:16:27.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:27.439306 24615 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:16:27.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:27.451854 24547 tablet.cc:2369] T 00000000000000000000000000000000 P 96e6a3a6c134457dab45358ef07a08e3: Can't schedule compaction. Clean time has not been advanced past its initial value.
16:16:27.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.428363 24517 master_runner.cc:186] Error getting master registration for 127.23.202.62:40007: OK, instance_id { permanent_uuid: "9b35ffacf87b4bbfb86b20c3359d87f2" instance_seqno: 1790439383613195 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
16:16:27.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.462636 24517 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:16:27.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.607517 24604 raft_consensus.cc:482] T 00000000000000000000000000000000 P 96e6a3a6c134457dab45358ef07a08e3 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
16:16:27.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.608079 24604 raft_consensus.cc:504] T 00000000000000000000000000000000 P 96e6a3a6c134457dab45358ef07a08e3 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9b35ffacf87b4bbfb86b20c3359d87f2" member_type: VOTER last_known_addr { host: "127.23.202.62" port: 40007 } } peers { permanent_uuid: "2a097e36445f4132acfe45358f72d2b9" member_type: VOTER last_known_addr { host: "127.23.202.61" port: 36223 } } peers { permanent_uuid: "96e6a3a6c134457dab45358ef07a08e3" member_type: VOTER last_known_addr { host: "127.23.202.60" port: 32791 } }
16:16:27.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.610766 24604 leader_election.cc:288] T 00000000000000000000000000000000 P 96e6a3a6c134457dab45358ef07a08e3 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 9b35ffacf87b4bbfb86b20c3359d87f2 (127.23.202.62:40007), 2a097e36445f4132acfe45358f72d2b9 (127.23.202.61:36223)
16:16:27.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.801891 24515 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.202.60" port: 32791 } attempt: 7
16:16:27.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.826205 24515 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2a097e36445f4132acfe45358f72d2b9: Bootstrap starting.
16:16:27.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.833652 24515 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 2a097e36445f4132acfe45358f72d2b9: Neither blocks nor log segments found. Creating new log.
16:16:27.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.835986 24515 log.cc:824] T 00000000000000000000000000000000 P 2a097e36445f4132acfe45358f72d2b9: Log is configured to *not* fsync() on all Append() calls
16:16:27.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.841310 24515 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2a097e36445f4132acfe45358f72d2b9: No bootstrap required, opened a new log
16:16:27.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.876040 24515 raft_consensus.cc:348] T 00000000000000000000000000000000 P 2a097e36445f4132acfe45358f72d2b9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9b35ffacf87b4bbfb86b20c3359d87f2" member_type: VOTER last_known_addr { host: "127.23.202.62" port: 40007 } } peers { permanent_uuid: "2a097e36445f4132acfe45358f72d2b9" member_type: VOTER last_known_addr { host: "127.23.202.61" port: 36223 } } peers { permanent_uuid: "96e6a3a6c134457dab45358ef07a08e3" member_type: VOTER last_known_addr { host: "127.23.202.60" port: 32791 } }
16:16:27.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.876971 24515 raft_consensus.cc:374] T 00000000000000000000000000000000 P 2a097e36445f4132acfe45358f72d2b9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
16:16:27.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.877249 24515 raft_consensus.cc:729] T 00000000000000000000000000000000 P 2a097e36445f4132acfe45358f72d2b9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2a097e36445f4132acfe45358f72d2b9, State: Initialized, Role: FOLLOWER
16:16:27.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.878180 24515 consensus_queue.cc:260] T 00000000000000000000000000000000 P 2a097e36445f4132acfe45358f72d2b9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9b35ffacf87b4bbfb86b20c3359d87f2" member_type: VOTER last_known_addr { host: "127.23.202.62" port: 40007 } } peers { permanent_uuid: "2a097e36445f4132acfe45358f72d2b9" member_type: VOTER last_known_addr { host: "127.23.202.61" port: 36223 } } peers { permanent_uuid: "96e6a3a6c134457dab45358ef07a08e3" member_type: VOTER last_known_addr { host: "127.23.202.60" port: 32791 } }
16:16:27.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.881239 24619 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2a097e36445f4132acfe45358f72d2b9 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9b35ffacf87b4bbfb86b20c3359d87f2" member_type: VOTER last_known_addr { host: "127.23.202.62" port: 40007 } } peers { permanent_uuid: "2a097e36445f4132acfe45358f72d2b9" member_type: VOTER last_known_addr { host: "127.23.202.61" port: 36223 } } peers { permanent_uuid: "96e6a3a6c134457dab45358ef07a08e3" member_type: VOTER last_known_addr { host: "127.23.202.60" port: 32791 } } }
16:16:27.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.882316 24619 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2a097e36445f4132acfe45358f72d2b9 [sys.catalog]: This master's current role is: FOLLOWER
16:16:27.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.883109 24515 sys_catalog.cc:564] T 00000000000000000000000000000000 P 2a097e36445f4132acfe45358f72d2b9 [sys.catalog]: configured and running, proceeding with master startup.
16:16:27.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.883350 24490 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "96e6a3a6c134457dab45358ef07a08e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2a097e36445f4132acfe45358f72d2b9" is_pre_election: true
16:16:27.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.884382 24490 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 2a097e36445f4132acfe45358f72d2b9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 96e6a3a6c134457dab45358ef07a08e3 in term 0.
16:16:27.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.885871 24543 leader_election.cc:302] T 00000000000000000000000000000000 P 96e6a3a6c134457dab45358ef07a08e3 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2a097e36445f4132acfe45358f72d2b9, 96e6a3a6c134457dab45358ef07a08e3; no voters:
16:16:27.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.886754 24604 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 96e6a3a6c134457dab45358ef07a08e3 [term 0 FOLLOWER]: Leader pre-election won for term 1
16:16:27.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.887095 24604 raft_consensus.cc:482] T 00000000000000000000000000000000 P 96e6a3a6c134457dab45358ef07a08e3 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
16:16:27.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.887435 24604 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 96e6a3a6c134457dab45358ef07a08e3 [term 0 FOLLOWER]: Advancing to term 1
16:16:27.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.895630 24604 raft_consensus.cc:504] T 00000000000000000000000000000000 P 96e6a3a6c134457dab45358ef07a08e3 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9b35ffacf87b4bbfb86b20c3359d87f2" member_type: VOTER last_known_addr { host: "127.23.202.62" port: 40007 } } peers { permanent_uuid: "2a097e36445f4132acfe45358f72d2b9" member_type: VOTER last_known_addr { host: "127.23.202.61" port: 36223 } } peers { permanent_uuid: "96e6a3a6c134457dab45358ef07a08e3" member_type: VOTER last_known_addr { host: "127.23.202.60" port: 32791 } }
16:16:27.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.899576 24490 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "96e6a3a6c134457dab45358ef07a08e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2a097e36445f4132acfe45358f72d2b9"
16:16:27.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.900188 24490 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 2a097e36445f4132acfe45358f72d2b9 [term 0 FOLLOWER]: Advancing to term 1
16:16:27.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.907745 24490 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 2a097e36445f4132acfe45358f72d2b9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 96e6a3a6c134457dab45358ef07a08e3 in term 1.
16:16:27.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.908886 24604 leader_election.cc:288] T 00000000000000000000000000000000 P 96e6a3a6c134457dab45358ef07a08e3 [CANDIDATE]: Term 1 election: Requested vote from peers 9b35ffacf87b4bbfb86b20c3359d87f2 (127.23.202.62:40007), 2a097e36445f4132acfe45358f72d2b9 (127.23.202.61:36223)
16:16:27.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.909297 24543 leader_election.cc:302] T 00000000000000000000000000000000 P 96e6a3a6c134457dab45358ef07a08e3 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2a097e36445f4132acfe45358f72d2b9, 96e6a3a6c134457dab45358ef07a08e3; no voters:
16:16:27.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.910043 24604 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 96e6a3a6c134457dab45358ef07a08e3 [term 1 FOLLOWER]: Leader election won for term 1
16:16:27.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.912283 24604 raft_consensus.cc:686] T 00000000000000000000000000000000 P 96e6a3a6c134457dab45358ef07a08e3 [term 1 LEADER]: Becoming Leader. State: Replica: 96e6a3a6c134457dab45358ef07a08e3, State: Running, Role: LEADER
16:16:27.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.913226 24604 consensus_queue.cc:237] T 00000000000000000000000000000000 P 96e6a3a6c134457dab45358ef07a08e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9b35ffacf87b4bbfb86b20c3359d87f2" member_type: VOTER last_known_addr { host: "127.23.202.62" port: 40007 } } peers { permanent_uuid: "2a097e36445f4132acfe45358f72d2b9" member_type: VOTER last_known_addr { host: "127.23.202.61" port: 36223 } } peers { permanent_uuid: "96e6a3a6c134457dab45358ef07a08e3" member_type: VOTER last_known_addr { host: "127.23.202.60" port: 32791 } }
16:16:27.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.931761 24626 sys_catalog.cc:455] T 00000000000000000000000000000000 P 96e6a3a6c134457dab45358ef07a08e3 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 96e6a3a6c134457dab45358ef07a08e3. Latest consensus state: current_term: 1 leader_uuid: "96e6a3a6c134457dab45358ef07a08e3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9b35ffacf87b4bbfb86b20c3359d87f2" member_type: VOTER last_known_addr { host: "127.23.202.62" port: 40007 } } peers { permanent_uuid: "2a097e36445f4132acfe45358f72d2b9" member_type: VOTER last_known_addr { host: "127.23.202.61" port: 36223 } } peers { permanent_uuid: "96e6a3a6c134457dab45358ef07a08e3" member_type: VOTER last_known_addr { host: "127.23.202.60" port: 32791 } } }
16:16:27.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.932575 24626 sys_catalog.cc:458] T 00000000000000000000000000000000 P 96e6a3a6c134457dab45358ef07a08e3 [sys.catalog]: This master's current role is: LEADER
16:16:27.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.944522 24632 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
16:16:27.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.949419 24633 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
16:16:27.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.952324 24632 catalog_manager.cc:1484] Initializing Kudu cluster ID...
16:16:27.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.969400 24490 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 2a097e36445f4132acfe45358f72d2b9 [term 1 FOLLOWER]: Refusing update from remote peer 96e6a3a6c134457dab45358ef07a08e3: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
16:16:27.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.971343 24626 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 96e6a3a6c134457dab45358ef07a08e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "2a097e36445f4132acfe45358f72d2b9" member_type: VOTER last_known_addr { host: "127.23.202.61" port: 36223 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
16:16:27.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.954119 24442 master_runner.cc:186] Error getting master registration for 127.23.202.62:40007: OK, instance_id { permanent_uuid: "9b35ffacf87b4bbfb86b20c3359d87f2" instance_seqno: 1790439383613195 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
16:16:27.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:27.997640 24442 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:16:28.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:28.005371 24619 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2a097e36445f4132acfe45358f72d2b9 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 96e6a3a6c134457dab45358ef07a08e3. Latest consensus state: current_term: 1 leader_uuid: "96e6a3a6c134457dab45358ef07a08e3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9b35ffacf87b4bbfb86b20c3359d87f2" member_type: VOTER last_known_addr { host: "127.23.202.62" port: 40007 } } peers { permanent_uuid: "2a097e36445f4132acfe45358f72d2b9" member_type: VOTER last_known_addr { host: "127.23.202.61" port: 36223 } } peers { permanent_uuid: "96e6a3a6c134457dab45358ef07a08e3" member_type: VOTER last_known_addr { host: "127.23.202.60" port: 32791 } } }
16:16:28.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:28.005955 24619 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2a097e36445f4132acfe45358f72d2b9 [sys.catalog]: This master's current role is: FOLLOWER
16:16:28.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:28.020128 24626 sys_catalog.cc:455] T 00000000000000000000000000000000 P 96e6a3a6c134457dab45358ef07a08e3 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "96e6a3a6c134457dab45358ef07a08e3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9b35ffacf87b4bbfb86b20c3359d87f2" member_type: VOTER last_known_addr { host: "127.23.202.62" port: 40007 } } peers { permanent_uuid: "2a097e36445f4132acfe45358f72d2b9" member_type: VOTER last_known_addr { host: "127.23.202.61" port: 36223 } } peers { permanent_uuid: "96e6a3a6c134457dab45358ef07a08e3" member_type: VOTER last_known_addr { host: "127.23.202.60" port: 32791 } } }
16:16:28.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:28.020932 24626 sys_catalog.cc:458] T 00000000000000000000000000000000 P 96e6a3a6c134457dab45358ef07a08e3 [sys.catalog]: This master's current role is: LEADER
16:16:28.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:28.022294 24619 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2a097e36445f4132acfe45358f72d2b9 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "96e6a3a6c134457dab45358ef07a08e3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9b35ffacf87b4bbfb86b20c3359d87f2" member_type: VOTER last_known_addr { host: "127.23.202.62" port: 40007 } } peers { permanent_uuid: "2a097e36445f4132acfe45358f72d2b9" member_type: VOTER last_known_addr { host: "127.23.202.61" port: 36223 } } peers { permanent_uuid: "96e6a3a6c134457dab45358ef07a08e3" member_type: VOTER last_known_addr { host: "127.23.202.60" port: 32791 } } }
16:16:28.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:28.023898 24632 catalog_manager.cc:1347] Generated new cluster ID: 9eb2914c7b8b45e9ae07c33a6e30377f
16:16:28.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:28.024384 24632 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
16:16:28.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:28.027529 24619 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2a097e36445f4132acfe45358f72d2b9 [sys.catalog]: This master's current role is: FOLLOWER
16:16:28.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:28.054131 24632 catalog_manager.cc:1370] Generated new certificate authority record
16:16:28.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:28.056932 24632 catalog_manager.cc:1504] Loading token signing keys...
16:16:28.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:28.077083 24632 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 96e6a3a6c134457dab45358ef07a08e3: Generated new TSK 0
16:16:28.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:28.078388 24632 catalog_manager.cc:1514] Initializing in-progress tserver states...
16:16:28.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:28.090000 24360 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskkW_qFj/build/tsan/bin/kudu
16:16:28.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskkW_qFj/build/tsan/bin/kudu
16:16:28.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/ts-0/wal
16:16:28.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/ts-0/data
16:16:28.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:16:28.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:16:28.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:16:28.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:16:28.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:16:28.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:16:28.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/ts-0/logs
16:16:28.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/ts-0/data/info.pb
16:16:28.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:16:28.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:16:28.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:16:28.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:16:28.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:16:28.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:16:28.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:16:28.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:16:28.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
16:16:28.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:16:28.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.202.1:0
16:16:28.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.202.1
16:16:28.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.202.1
16:16:28.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:16:28.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.202.62:40007,127.23.202.61:36223,127.23.202.60:32791
16:16:28.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.202.20:43197
16:16:28.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:16:28.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:16:28.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
16:16:28.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:28.280229 24440 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.202.60" port: 32791 } attempt: 7
16:16:28.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:28.320758 24440 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9b35ffacf87b4bbfb86b20c3359d87f2: Bootstrap starting.
16:16:28.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:28.329205 24440 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 9b35ffacf87b4bbfb86b20c3359d87f2: Neither blocks nor log segments found. Creating new log.
16:16:28.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:28.332072 24440 log.cc:824] T 00000000000000000000000000000000 P 9b35ffacf87b4bbfb86b20c3359d87f2: Log is configured to *not* fsync() on all Append() calls
16:16:28.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:28.343839 24440 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9b35ffacf87b4bbfb86b20c3359d87f2: No bootstrap required, opened a new log
16:16:28.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:28.380358 24440 raft_consensus.cc:348] T 00000000000000000000000000000000 P 9b35ffacf87b4bbfb86b20c3359d87f2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9b35ffacf87b4bbfb86b20c3359d87f2" member_type: VOTER last_known_addr { host: "127.23.202.62" port: 40007 } } peers { permanent_uuid: "2a097e36445f4132acfe45358f72d2b9" member_type: VOTER last_known_addr { host: "127.23.202.61" port: 36223 } } peers { permanent_uuid: "96e6a3a6c134457dab45358ef07a08e3" member_type: VOTER last_known_addr { host: "127.23.202.60" port: 32791 } }
16:16:28.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:28.381845 24440 raft_consensus.cc:374] T 00000000000000000000000000000000 P 9b35ffacf87b4bbfb86b20c3359d87f2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
16:16:28.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:28.382479 24440 raft_consensus.cc:729] T 00000000000000000000000000000000 P 9b35ffacf87b4bbfb86b20c3359d87f2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9b35ffacf87b4bbfb86b20c3359d87f2, State: Initialized, Role: FOLLOWER
16:16:28.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:28.383916 24440 consensus_queue.cc:260] T 00000000000000000000000000000000 P 9b35ffacf87b4bbfb86b20c3359d87f2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9b35ffacf87b4bbfb86b20c3359d87f2" member_type: VOTER last_known_addr { host: "127.23.202.62" port: 40007 } } peers { permanent_uuid: "2a097e36445f4132acfe45358f72d2b9" member_type: VOTER last_known_addr { host: "127.23.202.61" port: 36223 } } peers { permanent_uuid: "96e6a3a6c134457dab45358ef07a08e3" member_type: VOTER last_known_addr { host: "127.23.202.60" port: 32791 } }
16:16:28.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:28.396378 24642 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9b35ffacf87b4bbfb86b20c3359d87f2 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9b35ffacf87b4bbfb86b20c3359d87f2" member_type: VOTER last_known_addr { host: "127.23.202.62" port: 40007 } } peers { permanent_uuid: "2a097e36445f4132acfe45358f72d2b9" member_type: VOTER last_known_addr { host: "127.23.202.61" port: 36223 } } peers { permanent_uuid: "96e6a3a6c134457dab45358ef07a08e3" member_type: VOTER last_known_addr { host: "127.23.202.60" port: 32791 } } }
16:16:28.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:28.403532 24642 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9b35ffacf87b4bbfb86b20c3359d87f2 [sys.catalog]: This master's current role is: FOLLOWER
16:16:28.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:28.399274 24413 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 9b35ffacf87b4bbfb86b20c3359d87f2 [term 0 FOLLOWER]: Advancing to term 1
16:16:28.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:28.399612 24415 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "96e6a3a6c134457dab45358ef07a08e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9b35ffacf87b4bbfb86b20c3359d87f2" is_pre_election: true
16:16:28.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:28.400187 24414 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "96e6a3a6c134457dab45358ef07a08e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9b35ffacf87b4bbfb86b20c3359d87f2"
16:16:28.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:28.398693 24440 sys_catalog.cc:564] T 00000000000000000000000000000000 P 9b35ffacf87b4bbfb86b20c3359d87f2 [sys.catalog]: configured and running, proceeding with master startup.
16:16:28.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:28.416581 24413 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 9b35ffacf87b4bbfb86b20c3359d87f2 [term 1 FOLLOWER]: Refusing update from remote peer 96e6a3a6c134457dab45358ef07a08e3: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
16:16:28.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:28.420904 24604 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 96e6a3a6c134457dab45358ef07a08e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9b35ffacf87b4bbfb86b20c3359d87f2" member_type: VOTER last_known_addr { host: "127.23.202.62" port: 40007 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
16:16:28.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:28.486361 24648 mvcc.cc:205] Tried to move back new op lower bound from 7333639733105647616 to 7333639732908437504. Current Snapshot: MvccSnapshot[applied={T|T < 7333639733105647616}]
16:16:28.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:28.509141 24642 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9b35ffacf87b4bbfb86b20c3359d87f2 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 96e6a3a6c134457dab45358ef07a08e3. Latest consensus state: current_term: 1 leader_uuid: "96e6a3a6c134457dab45358ef07a08e3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9b35ffacf87b4bbfb86b20c3359d87f2" member_type: VOTER last_known_addr { host: "127.23.202.62" port: 40007 } } peers { permanent_uuid: "2a097e36445f4132acfe45358f72d2b9" member_type: VOTER last_known_addr { host: "127.23.202.61" port: 36223 } } peers { permanent_uuid: "96e6a3a6c134457dab45358ef07a08e3" member_type: VOTER last_known_addr { host: "127.23.202.60" port: 32791 } } }
16:16:28.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:28.510900 24642 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9b35ffacf87b4bbfb86b20c3359d87f2 [sys.catalog]: This master's current role is: FOLLOWER
16:16:28.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:28.508208 24644 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9b35ffacf87b4bbfb86b20c3359d87f2 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "96e6a3a6c134457dab45358ef07a08e3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9b35ffacf87b4bbfb86b20c3359d87f2" member_type: VOTER last_known_addr { host: "127.23.202.62" port: 40007 } } peers { permanent_uuid: "2a097e36445f4132acfe45358f72d2b9" member_type: VOTER last_known_addr { host: "127.23.202.61" port: 36223 } } peers { permanent_uuid: "96e6a3a6c134457dab45358ef07a08e3" member_type: VOTER last_known_addr { host: "127.23.202.60" port: 32791 } } }
16:16:28.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:28.511849 24644 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9b35ffacf87b4bbfb86b20c3359d87f2 [sys.catalog]: This master's current role is: FOLLOWER
16:16:28.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:28.555706 24662 catalog_manager.cc:1259] Loaded cluster ID: 9eb2914c7b8b45e9ae07c33a6e30377f
16:16:28.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:28.556324 24662 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 9b35ffacf87b4bbfb86b20c3359d87f2: loading cluster ID for follower catalog manager: success
16:16:28.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:28.564219 24662 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 9b35ffacf87b4bbfb86b20c3359d87f2: acquiring CA information for follower catalog manager: success
16:16:28.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:28.569898 24662 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 9b35ffacf87b4bbfb86b20c3359d87f2: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
16:16:28.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:28.579167 24639 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:16:28.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:28.579775 24639 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:16:28.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:28.580420 24639 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:16:28.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:28.614854 24639 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:16:28.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:28.616117 24639 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.202.1
16:16:28.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:28.674726 24639 tablet_server_runner.cc:78] Tablet server non-default flags:
16:16:28.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:16:28.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.202.20:43197
16:16:28.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:16:28.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:16:28.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/ts-0/data
16:16:28.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/ts-0/wal
16:16:28.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:16:28.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:16:28.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.202.1:0
16:16:28.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:16:28.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:16:28.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:16:28.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/ts-0/data/info.pb
16:16:28.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.202.1
16:16:28.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:16:28.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.202.62:40007,127.23.202.61:36223,127.23.202.60:32791
16:16:28.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:16:28.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:16:28.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:16:28.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:16:28.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:16:28.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.202.1
16:16:28.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/ts-0/logs
16:16:28.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:16:28.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:16:28.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:16:28.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
16:16:28.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
16:16:28.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
16:16:28.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:16:28.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 14:44:58 UTC on ccycloud.quasar-hxpbms.root.comops.site
16:16:28.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4674114
16:16:28.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:16:28.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:28.678826 24639 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:16:28.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:28.681227 24639 file_cache.cc:493] Constructed file cache file cache with capacity 419430
16:16:28.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:28.719678 24665 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:16:28.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:28.902184 24604 sys_catalog.cc:455] T 00000000000000000000000000000000 P 96e6a3a6c134457dab45358ef07a08e3 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "96e6a3a6c134457dab45358ef07a08e3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9b35ffacf87b4bbfb86b20c3359d87f2" member_type: VOTER last_known_addr { host: "127.23.202.62" port: 40007 } } peers { permanent_uuid: "2a097e36445f4132acfe45358f72d2b9" member_type: VOTER last_known_addr { host: "127.23.202.61" port: 36223 } } peers { permanent_uuid: "96e6a3a6c134457dab45358ef07a08e3" member_type: VOTER last_known_addr { host: "127.23.202.60" port: 32791 } } }
16:16:28.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:28.903496 24604 sys_catalog.cc:458] T 00000000000000000000000000000000 P 96e6a3a6c134457dab45358ef07a08e3 [sys.catalog]: This master's current role is: LEADER
16:16:28.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:28.960078 24633 catalog_manager.cc:1259] Loaded cluster ID: 9eb2914c7b8b45e9ae07c33a6e30377f
16:16:28.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:28.967571 24633 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 2a097e36445f4132acfe45358f72d2b9: loading cluster ID for follower catalog manager: success
16:16:28.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:28.976053 24633 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 2a097e36445f4132acfe45358f72d2b9: acquiring CA information for follower catalog manager: success
16:16:28.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:28.993346 24633 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 2a097e36445f4132acfe45358f72d2b9: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
16:16:29.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:29.834318 24591 debug-util.cc:398] Leaking SignalData structure 0x7b0800072220 after lost signal to thread 24522
16:16:29.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:29.835016 24591 debug-util.cc:398] Leaking SignalData structure 0x7b08000c1980 after lost signal to thread 24594
16:16:30.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:28.720165 24666 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:16:30.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:28.724499 24668 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:16:30.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:30.284644 24667 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
16:16:30.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:30.285228 24639 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:16:30.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:30.289294 24639 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:16:30.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:30.299778 24639 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:16:30.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:30.301512 24639 hybrid_clock.cc:648] HybridClock initialized: now 1790439390301441 us; error 86 us; skew 500 ppm
16:16:30.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:30.302608 24639 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:16:30.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:30.326514 24639 webserver.cc:460] Webserver started at http://127.23.202.1:41061/ using document root <none> and password file <none>
16:16:30.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:30.328173 24639 fs_manager.cc:359] Metadata directory not provided
16:16:30.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:30.328732 24639 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:16:30.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:30.329488 24639 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
16:16:30.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:30.337386 24639 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/ts-0/data/instance:
16:16:30.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4396a3c87d954f8bb9443e5c90424e0b"
16:16:30.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:16:30 on dist-test-slave-fnz7"
16:16:30.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:30.340601 24639 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/ts-0/wal/instance:
16:16:30.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4396a3c87d954f8bb9443e5c90424e0b"
16:16:30.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:16:30 on dist-test-slave-fnz7"
16:16:30.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:30.380908 24639 fs_manager.cc:691] Time spent creating directory manager: real 0.036s user 0.010s sys 0.000s
16:16:30.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:30.396446 24678 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:16:30.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:30.402269 24639 fs_manager.cc:725] Time spent opening block manager: real 0.016s user 0.005s sys 0.003s
16:16:30.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:30.402925 24639 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/ts-0/data,/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/ts-0/wal
16:16:30.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4396a3c87d954f8bb9443e5c90424e0b"
16:16:30.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:16:30 on dist-test-slave-fnz7"
16:16:30.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:30.403620 24639 fs_report.cc:383] FS layout report
16:16:30.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:16:30.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/ts-0/wal
16:16:30.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/ts-0/wal
16:16:30.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/ts-0/data/data
16:16:30.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:16:30.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:16:30.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:16:30.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:16:30.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:16:30.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:16:30.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:16:30.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:16:30.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:16:30.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:16:30.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:16:30.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:30.531548 24639 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
16:16:30.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:30.534205 24639 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:16:30.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:30.535144 24639 kserver.cc:163] Server-wide thread pool size limit: 3276
16:16:30.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:30.551559 24639 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
16:16:30.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:30.569238 24639 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
16:16:30.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:30.569607 24639 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
16:16:30.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:30.569900 24639 ts_tablet_manager.cc:610] Registered 0 tablets
16:16:30.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:30.570101 24639 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
16:16:31.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:31.520004 24792 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.23.202.1:36735 every 8 connection(s)
16:16:31.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:31.520972 24639 rpc_server.cc:307] RPC server started. Bound to: 127.23.202.1:36735
16:16:31.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:31.525429 24639 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/ts-0/data/info.pb
16:16:31.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:31.526623 24360 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskkW_qFj/build/tsan/bin/kudu as pid 24639
16:16:31.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:31.527089 24360 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/ts-0/wal/instance
16:16:31.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:31.544504 24360 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskkW_qFj/build/tsan/bin/kudu
16:16:31.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskkW_qFj/build/tsan/bin/kudu
16:16:31.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/ts-1/wal
16:16:31.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/ts-1/data
16:16:31.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:16:31.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:16:31.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:16:31.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:16:31.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:16:31.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:16:31.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/ts-1/logs
16:16:31.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/ts-1/data/info.pb
16:16:31.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:16:31.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:16:31.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:16:31.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:16:31.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:16:31.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:16:31.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:16:31.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:16:31.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
16:16:31.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:16:31.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.202.2:0
16:16:31.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.202.2
16:16:31.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.202.2
16:16:31.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:16:31.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.202.62:40007,127.23.202.61:36223,127.23.202.60:32791
16:16:31.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.202.20:43197
16:16:31.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:16:31.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:16:31.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
16:16:31.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:31.690147 24797 heartbeater.cc:347] Connected to a master server at 127.23.202.62:40007
16:16:31.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:31.690620 24797 heartbeater.cc:464] Registering TS with master...
16:16:31.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:31.691848 24797 heartbeater.cc:511] Master 127.23.202.62:40007 requested a full tablet report, sending...
16:16:31.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:31.694293 24794 heartbeater.cc:347] Connected to a master server at 127.23.202.60:32791
16:16:31.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:31.694648 24794 heartbeater.cc:464] Registering TS with master...
16:16:31.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:31.695441 24794 heartbeater.cc:511] Master 127.23.202.60:32791 requested a full tablet report, sending...
16:16:31.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:31.698009 24558 ts_manager.cc:194] Registered new tserver with Master: 4396a3c87d954f8bb9443e5c90424e0b (127.23.202.1:36735)
16:16:31.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:31.698400 24405 ts_manager.cc:194] Registered new tserver with Master: 4396a3c87d954f8bb9443e5c90424e0b (127.23.202.1:36735)
16:16:31.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:31.700919 24558 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.23.202.1:51177
16:16:31.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:31.702373 24795 heartbeater.cc:347] Connected to a master server at 127.23.202.61:36223
16:16:31.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:31.702741 24795 heartbeater.cc:464] Registering TS with master...
16:16:31.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:31.703599 24795 heartbeater.cc:511] Master 127.23.202.61:36223 requested a full tablet report, sending...
16:16:31.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:31.707597 24480 ts_manager.cc:194] Registered new tserver with Master: 4396a3c87d954f8bb9443e5c90424e0b (127.23.202.1:36735)
16:16:31.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:31.840059 24511 debug-util.cc:398] Leaking SignalData structure 0x7b0800072900 after lost signal to thread 24446
16:16:32.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:32.153241 24793 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:16:32.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:32.153973 24793 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:16:32.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:32.154841 24793 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:16:32.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:32.225598 24793 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:16:32.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:32.227242 24793 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.202.2
16:16:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:32.308729 24793 tablet_server_runner.cc:78] Tablet server non-default flags:
16:16:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:16:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.202.20:43197
16:16:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:16:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:16:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/ts-1/data
16:16:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/ts-1/wal
16:16:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:16:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:16:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.202.2:0
16:16:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:16:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:16:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:16:32.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/ts-1/data/info.pb
16:16:32.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.202.2
16:16:32.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:16:32.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.202.62:40007,127.23.202.61:36223,127.23.202.60:32791
16:16:32.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:16:32.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:16:32.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:16:32.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:16:32.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:16:32.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.202.2
16:16:32.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/ts-1/logs
16:16:32.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:16:32.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:16:32.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:16:32.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
16:16:32.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
16:16:32.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
16:16:32.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:16:32.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 14:44:58 UTC on ccycloud.quasar-hxpbms.root.comops.site
16:16:32.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4674114
16:16:32.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:16:32.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:32.310527 24793 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:16:32.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:32.312870 24793 file_cache.cc:493] Constructed file cache file cache with capacity 419430
16:16:32.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:32.376389 24812 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:16:32.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:32.708711 24794 heartbeater.cc:503] Master 127.23.202.60:32791 was elected leader, sending a full tablet report...
16:16:33.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:32.384402 24793 server_base.cc:1034] running on GCE node
16:16:33.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:32.379237 24814 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:16:33.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:32.378381 24811 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:16:33.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:33.883545 24793 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:16:33.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:33.886816 24793 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:16:33.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:33.888619 24793 hybrid_clock.cc:648] HybridClock initialized: now 1790439393888570 us; error 68 us; skew 500 ppm
16:16:33.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:33.889585 24793 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:16:33.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:33.897675 24793 webserver.cc:460] Webserver started at http://127.23.202.2:46621/ using document root <none> and password file <none>
16:16:33.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:33.898783 24793 fs_manager.cc:359] Metadata directory not provided
16:16:33.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:33.899565 24793 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:16:33.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:33.900331 24793 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
16:16:33.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:33.906046 24793 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/ts-1/data/instance:
16:16:33.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "937e56fa15124fe4a56267d6a7339dd2"
16:16:33.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:16:33 on dist-test-slave-fnz7"
16:16:33.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:33.907500 24793 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/ts-1/wal/instance:
16:16:33.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "937e56fa15124fe4a56267d6a7339dd2"
16:16:33.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:16:33 on dist-test-slave-fnz7"
16:16:33.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:33.916285 24793 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.007s sys 0.000s
16:16:33.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:33.923653 24823 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:16:33.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:33.924962 24793 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.003s sys 0.001s
16:16:33.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:33.925279 24793 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/ts-1/data,/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/ts-1/wal
16:16:33.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "937e56fa15124fe4a56267d6a7339dd2"
16:16:33.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:16:33 on dist-test-slave-fnz7"
16:16:33.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:33.925602 24793 fs_report.cc:383] FS layout report
16:16:33.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:16:33.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/ts-1/wal
16:16:33.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/ts-1/wal
16:16:33.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/ts-1/data/data
16:16:33.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:16:33.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:16:33.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:16:33.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:16:33.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:16:33.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:16:33.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:16:33.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:16:33.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:16:33.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:16:33.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:16:34.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:34.008661 24793 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
16:16:34.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:34.010841 24793 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:16:34.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:34.011691 24793 kserver.cc:163] Server-wide thread pool size limit: 3276
16:16:34.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:34.016352 24793 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
16:16:34.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:34.023192 24793 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
16:16:34.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:34.023574 24793 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.001s sys 0.000s
16:16:34.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:34.023913 24793 ts_tablet_manager.cc:610] Registered 0 tablets
16:16:34.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:34.024163 24793 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
16:16:34.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:34.216843 24793 rpc_server.cc:307] RPC server started. Bound to: 127.23.202.2:42683
16:16:34.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:34.217005 24937 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.23.202.2:42683 every 8 connection(s)
16:16:34.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:34.219796 24793 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/ts-1/data/info.pb
16:16:34.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:34.223906 24360 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskkW_qFj/build/tsan/bin/kudu as pid 24793
16:16:34.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:34.224637 24360 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/ts-1/wal/instance
16:16:34.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:34.257367 24360 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskkW_qFj/build/tsan/bin/kudu
16:16:34.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskkW_qFj/build/tsan/bin/kudu
16:16:34.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/ts-2/wal
16:16:34.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/ts-2/data
16:16:34.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:16:34.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:16:34.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:16:34.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:16:34.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:16:34.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:16:34.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/ts-2/logs
16:16:34.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/ts-2/data/info.pb
16:16:34.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:16:34.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:16:34.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:16:34.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:16:34.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:16:34.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:16:34.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:16:34.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:16:34.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
16:16:34.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:16:34.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.202.3:0
16:16:34.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.202.3
16:16:34.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.202.3
16:16:34.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:16:34.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.202.62:40007,127.23.202.61:36223,127.23.202.60:32791
16:16:34.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.202.20:43197
16:16:34.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:16:34.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:16:34.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
16:16:34.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:34.261634 24938 heartbeater.cc:347] Connected to a master server at 127.23.202.60:32791
16:16:34.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:34.262179 24938 heartbeater.cc:464] Registering TS with master...
16:16:34.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:34.263561 24938 heartbeater.cc:511] Master 127.23.202.60:32791 requested a full tablet report, sending...
16:16:34.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:34.267215 24558 ts_manager.cc:194] Registered new tserver with Master: 937e56fa15124fe4a56267d6a7339dd2 (127.23.202.2:42683)
16:16:34.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:34.269711 24558 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.23.202.2:51559
16:16:34.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:34.279796 24942 heartbeater.cc:347] Connected to a master server at 127.23.202.62:40007
16:16:34.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:34.280208 24942 heartbeater.cc:464] Registering TS with master...
16:16:34.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:34.280578 24939 heartbeater.cc:347] Connected to a master server at 127.23.202.61:36223
16:16:34.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:34.280916 24939 heartbeater.cc:464] Registering TS with master...
16:16:34.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:34.280963 24942 heartbeater.cc:511] Master 127.23.202.62:40007 requested a full tablet report, sending...
16:16:34.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:34.281597 24939 heartbeater.cc:511] Master 127.23.202.61:36223 requested a full tablet report, sending...
16:16:34.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:34.283844 24405 ts_manager.cc:194] Registered new tserver with Master: 937e56fa15124fe4a56267d6a7339dd2 (127.23.202.2:42683)
16:16:34.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:34.284556 24480 ts_manager.cc:194] Registered new tserver with Master: 937e56fa15124fe4a56267d6a7339dd2 (127.23.202.2:42683)
16:16:34.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:34.598511 24948 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:16:34.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:34.599256 24948 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:16:34.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:34.600221 24948 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:16:34.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:34.641466 24948 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:16:34.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:34.642825 24948 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.202.3
16:16:34.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:34.692101 24948 tablet_server_runner.cc:78] Tablet server non-default flags:
16:16:34.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:16:34.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.202.20:43197
16:16:34.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:16:34.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:16:34.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/ts-2/data
16:16:34.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/ts-2/wal
16:16:34.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:16:34.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:16:34.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.202.3:0
16:16:34.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:16:34.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:16:34.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:16:34.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/ts-2/data/info.pb
16:16:34.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.202.3
16:16:34.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:16:34.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.202.62:40007,127.23.202.61:36223,127.23.202.60:32791
16:16:34.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:16:34.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:16:34.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:16:34.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:16:34.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:16:34.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.202.3
16:16:34.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/ts-2/logs
16:16:34.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:16:34.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:16:34.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:16:34.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
16:16:34.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
16:16:34.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
16:16:34.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:16:34.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 14:44:58 UTC on ccycloud.quasar-hxpbms.root.comops.site
16:16:34.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4674114
16:16:34.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:16:34.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:34.693560 24948 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:16:34.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:34.695326 24948 file_cache.cc:493] Constructed file cache file cache with capacity 419430
16:16:34.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:34.715937 24954 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:16:35.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:35.275674 24938 heartbeater.cc:503] Master 127.23.202.60:32791 was elected leader, sending a full tablet report...
16:16:35.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:34.715952 24955 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:16:35.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:34.717896 24948 server_base.cc:1034] running on GCE node
16:16:35.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:34.716156 24957 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:16:35.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:35.936846 24948 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:16:35.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:35.939085 24948 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:16:35.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:35.940606 24948 hybrid_clock.cc:648] HybridClock initialized: now 1790439395940566 us; error 48 us; skew 500 ppm
16:16:35.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:35.941754 24948 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:16:35.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:35.955762 24948 webserver.cc:460] Webserver started at http://127.23.202.3:40035/ using document root <none> and password file <none>
16:16:35.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:35.956688 24948 fs_manager.cc:359] Metadata directory not provided
16:16:35.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:35.956878 24948 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:16:35.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:35.957296 24948 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
16:16:35.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:35.962716 24948 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/ts-2/data/instance:
16:16:35.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5ac888cefde247a4baaa15e8ce5b91c2"
16:16:35.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:16:35 on dist-test-slave-fnz7"
16:16:35.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:35.963867 24948 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/ts-2/wal/instance:
16:16:35.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5ac888cefde247a4baaa15e8ce5b91c2"
16:16:35.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:16:35 on dist-test-slave-fnz7"
16:16:35.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:35.971666 24948 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
16:16:35.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:35.977654 24965 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:16:35.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:35.979053 24948 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.001s
16:16:35.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:35.979575 24948 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/ts-2/data,/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/ts-2/wal
16:16:35.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5ac888cefde247a4baaa15e8ce5b91c2"
16:16:35.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:16:35 on dist-test-slave-fnz7"
16:16:35.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:35.979957 24948 fs_report.cc:383] FS layout report
16:16:35.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:16:35.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/ts-2/wal
16:16:35.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/ts-2/wal
16:16:35.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/ts-2/data/data
16:16:35.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:16:35.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:16:35.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:16:35.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:16:35.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:16:35.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:16:35.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:16:35.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:16:35.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:16:35.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:16:35.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:16:36.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:36.016748 24948 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
16:16:36.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:36.018307 24948 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:16:36.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:36.018779 24948 kserver.cc:163] Server-wide thread pool size limit: 3276
16:16:36.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:36.021420 24948 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
16:16:36.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:36.025935 24948 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
16:16:36.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:36.026147 24948 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
16:16:36.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:36.026360 24948 ts_tablet_manager.cc:610] Registered 0 tablets
16:16:36.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:36.026494 24948 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
16:16:36.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:36.170308 24948 rpc_server.cc:307] RPC server started. Bound to: 127.23.202.3:34669
16:16:36.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:36.170437 25077 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.23.202.3:34669 every 8 connection(s)
16:16:36.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:36.173817 24948 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/ts-2/data/info.pb
16:16:36.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:36.178853 24360 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskkW_qFj/build/tsan/bin/kudu as pid 24948
16:16:36.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:36.179355 24360 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster6641385230026048175/ts-2/wal/instance
16:16:36.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:36.237334 25078 heartbeater.cc:347] Connected to a master server at 127.23.202.60:32791
16:16:36.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:36.237780 25078 heartbeater.cc:464] Registering TS with master...
16:16:36.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:36.239060 25078 heartbeater.cc:511] Master 127.23.202.60:32791 requested a full tablet report, sending...
16:16:36.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:36.242396 24557 ts_manager.cc:194] Registered new tserver with Master: 5ac888cefde247a4baaa15e8ce5b91c2 (127.23.202.3:34669)
16:16:36.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:36.244736 25079 heartbeater.cc:347] Connected to a master server at 127.23.202.61:36223
16:16:36.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:36.244957 24557 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.23.202.3:41279
16:16:36.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:36.245069 25079 heartbeater.cc:464] Registering TS with master...
16:16:36.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:36.245980 25079 heartbeater.cc:511] Master 127.23.202.61:36223 requested a full tablet report, sending...
16:16:36.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:36.248925 24479 ts_manager.cc:194] Registered new tserver with Master: 5ac888cefde247a4baaa15e8ce5b91c2 (127.23.202.3:34669)
16:16:36.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:36.251924 25080 heartbeater.cc:347] Connected to a master server at 127.23.202.62:40007
16:16:36.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:36.252185 25080 heartbeater.cc:464] Registering TS with master...
16:16:36.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:36.252665 25080 heartbeater.cc:511] Master 127.23.202.62:40007 requested a full tablet report, sending...
16:16:36.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:36.254709 24405 ts_manager.cc:194] Registered new tserver with Master: 5ac888cefde247a4baaa15e8ce5b91c2 (127.23.202.3:34669)
16:16:36.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:36.272291 24360 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
16:16:36.283 [DEBUG - main] (MiniKuduCluster.java:195) Response:
16:16:36.284 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
16:16:36.295 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.23.202.62"
port: 40007
}
bound_http_address {
host: "127.23.202.62"
port: 41375
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.23.202.61"
port: 36223
}
bound_http_address {
host: "127.23.202.61"
port: 35971
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.23.202.60"
port: 32791
}
bound_http_address {
host: "127.23.202.60"
port: 41551
}
}
}
16:16:36.302 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
16:16:36.306 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.23.202.1"
port: 36735
}
bound_http_address {
host: "127.23.202.1"
port: 41061
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.23.202.2"
port: 42683
}
bound_http_address {
host: "127.23.202.2"
port: 46621
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.23.202.3"
port: 34669
}
bound_http_address {
host: "127.23.202.3"
port: 40035
}
}
}
16:16:36.306 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
16:16:36.489 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
16:16:36.619 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskkW_qFj/test-tmp
16:16:36.620 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
16:16:36.620 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskkW_qFj/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
16:16:36.626 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
16:16:36.978 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:16:36.978 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
16:16:36.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:36.981709 25089 mini_oidc.cc:152] Starting JWKS server
16:16:36.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:36.985993 25089 webserver.cc:460] Webserver started at http://127.0.0.1:38553/ using document root <none> and password file <none>
16:16:36.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:36.986447 25089 mini_oidc.cc:189] Starting OIDC Discovery server
16:16:36.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:36.988260 25089 webserver.cc:460] Webserver started at http://127.0.0.1:40323/ using document root <none> and password file <none>
16:16:37.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
16:16:37.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/krb5kdc/principal' for realm 'KRBTEST.COM',
16:16:37.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
16:16:37.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:16:37 dist-test-slave-fnz7 krb5kdc[25101](info): setting up network...
16:16:37.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
16:16:37.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:16:37 dist-test-slave-fnz7 krb5kdc[25101](info): set up 2 sockets
16:16:37.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:16:37 dist-test-slave-fnz7 krb5kdc[25101](info): commencing operation
16:16:37.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
16:16:37.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:37.250070 25078 heartbeater.cc:503] Master 127.23.202.60:32791 was elected leader, sending a full tablet report...
16:16:38.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:38.127627 25089 mini_kdc.cc:121] Time spent starting KDC: real 1.111s user 0.003s sys 0.018s
16:16:38.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
16:16:38.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
16:16:38.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
16:16:38.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
16:16:38.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
16:16:38.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
16:16:38.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
16:16:38.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
16:16:38.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
16:16:38.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
16:16:38.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/krb5kdc/test-user.keytab.
16:16:38.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/krb5kdc/test-user.keytab.
16:16:38.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:16:38 dist-test-slave-fnz7 krb5kdc[25101](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790439398, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:16:38.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
16:16:38.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-26T16:16:38Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
16:16:38.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-26T16:16:38Z Disabled control of system clock
16:16:38.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.24.128.126@KRBTEST.COM; defaulting to no policy
16:16:38.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:16:38.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.24.128.126@KRBTEST.COM" created.
16:16:38.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:16:38.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.24.128.126 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/krb5kdc/kudu_127.24.128.126.keytab.
16:16:38.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.24.128.126 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/krb5kdc/kudu_127.24.128.126.keytab.
16:16:38.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:38.337157 25089 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskkW_qFj/build/tsan/bin/kudu
16:16:38.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskkW_qFj/build/tsan/bin/kudu
16:16:38.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-0/wal
16:16:38.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-0/data
16:16:38.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:16:38.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:16:38.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:16:38.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:16:38.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:16:38.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:16:38.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-0/logs
16:16:38.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-0/data/info.pb
16:16:38.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:16:38.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:16:38.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:16:38.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:16:38.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:16:38.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:16:38.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:16:38.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:16:38.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:16:38.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:16:38.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:16:38.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:16:38.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.128.126:46405
16:16:38.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.128.126
16:16:38.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:16:38.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/krb5kdc/kudu_127.24.128.126.keytab
16:16:38.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.128.126
16:16:38.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:16:38.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:16:38.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:16:38.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.128.84:35767
16:16:38.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:16:38.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:16:38.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:16:38.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:16:38.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.128.126:46405,127.24.128.125:43165,127.24.128.124:43927
16:16:38.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:16:38.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38553/jwks.json
16:16:38.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:16:38.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:38.654093 25114 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:16:38.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:38.654654 25114 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:16:38.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:38.655459 25114 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:16:38.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:38.687866 25114 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
16:16:38.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:38.688108 25114 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:16:38.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:38.688477 25114 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:16:38.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:38.688804 25114 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:16:38.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:38.689100 25114 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:16:38.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:38.689409 25114 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:16:38.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:38.689569 25114 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38553/jwks.json
16:16:38.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:38.729063 25114 master_runner.cc:386] Master server non-default flags:
16:16:38.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:16:38.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.128.84:35767
16:16:38.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:16:38.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:16:38.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-0/data
16:16:38.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-0/wal
16:16:38.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
16:16:38.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:16:38.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.128.126:46405,127.24.128.125:43165,127.24.128.124:43927
16:16:38.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/krb5kdc/kudu_127.24.128.126.keytab
16:16:38.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.128.126
16:16:38.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:16:38.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:16:38.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:16:38.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.128.126:46405
16:16:38.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:16:38.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:16:38.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:16:38.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38553/jwks.json
16:16:38.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:16:38.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:16:38.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:16:38.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:16:38.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:16:38.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-0/data/info.pb
16:16:38.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.128.126
16:16:38.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:16:38.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:16:38.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:16:38.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:16:38.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:16:38.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:16:38.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-0/logs
16:16:38.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:16:38.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:16:38.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:16:38.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:16:38.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
16:16:38.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
16:16:38.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:16:38.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 14:44:58 UTC on ccycloud.quasar-hxpbms.root.comops.site
16:16:38.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4674114
16:16:38.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:16:38.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:38.730250 25114 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:16:38.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:38.731206 25114 file_cache.cc:493] Constructed file cache file cache with capacity 419430
16:16:38.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:38.752171 25121 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:16:39.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:38.752247 25122 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:16:39.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:38.753994 25114 server_base.cc:1034] running on GCE node
16:16:39.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:38.752207 25124 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:16:40.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:40.000377 25114 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:16:40.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:40.003716 25114 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:16:40.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:40.005286 25114 hybrid_clock.cc:648] HybridClock initialized: now 1790439400005242 us; error 47 us; skew 500 ppm
16:16:40.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:40.006465 25114 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:16:40.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:16:40 dist-test-slave-fnz7 krb5kdc[25101](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790439400, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.128.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:16:40.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:40.013396 25114 init.cc:377] Logged in from keytab as kudu/127.24.128.126@KRBTEST.COM (short username kudu)
16:16:40.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:40.018790 25114 webserver.cc:460] Webserver started at http://127.24.128.126:45197/ using document root <none> and password file <none>
16:16:40.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:40.019920 25114 fs_manager.cc:359] Metadata directory not provided
16:16:40.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:40.020215 25114 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:16:40.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:40.020781 25114 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
16:16:40.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:40.026284 25114 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-0/data/instance:
16:16:40.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1a14e6a5198641059abfde130f5ae6ff"
16:16:40.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:16:40 on dist-test-slave-fnz7"
16:16:40.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:40.027647 25114 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-0/wal/instance:
16:16:40.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1a14e6a5198641059abfde130f5ae6ff"
16:16:40.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:16:40 on dist-test-slave-fnz7"
16:16:40.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:40.035199 25114 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.002s sys 0.005s
16:16:40.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:40.041018 25132 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:16:40.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:40.042228 25114 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
16:16:40.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:40.042521 25114 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-0/data,/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-0/wal
16:16:40.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1a14e6a5198641059abfde130f5ae6ff"
16:16:40.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:16:40 on dist-test-slave-fnz7"
16:16:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:40.042866 25114 fs_report.cc:383] FS layout report
16:16:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:16:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-0/wal
16:16:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-0/wal
16:16:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-0/data/data
16:16:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:16:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:16:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:16:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:16:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:16:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:16:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:16:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:16:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:16:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:16:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:16:40.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:40.080041 25114 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
16:16:40.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:40.086122 25114 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:16:40.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:40.086570 25114 kserver.cc:163] Server-wide thread pool size limit: 3276
16:16:40.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:40.163275 25114 rpc_server.cc:307] RPC server started. Bound to: 127.24.128.126:46405
16:16:40.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:40.163506 25183 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.24.128.126:46405 every 8 connection(s)
16:16:40.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:40.166112 25114 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-0/data/info.pb
16:16:40.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:40.174310 25184 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
16:16:40.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:40.176751 25089 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskkW_qFj/build/tsan/bin/kudu as pid 25114
16:16:40.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:40.177461 25089 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-0/wal/instance
16:16:40.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.24.128.125@KRBTEST.COM; defaulting to no policy
16:16:40.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:16:40.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.24.128.125@KRBTEST.COM" created.
16:16:40.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:40.204661 25184 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.128.126" port: 46405 } has no permanent_uuid. Determining permanent_uuid...
16:16:40.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:16:40.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.24.128.125 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/krb5kdc/kudu_127.24.128.125.keytab.
16:16:40.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.24.128.125 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/krb5kdc/kudu_127.24.128.125.keytab.
16:16:40.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:40.237056 25089 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskkW_qFj/build/tsan/bin/kudu
16:16:40.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskkW_qFj/build/tsan/bin/kudu
16:16:40.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-1/wal
16:16:40.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-1/data
16:16:40.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:16:40.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:16:40.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:16:40.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:16:40.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:16:40.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:16:40.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-1/logs
16:16:40.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-1/data/info.pb
16:16:40.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:16:40.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:16:40.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:16:40.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:16:40.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:16:40.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:16:40.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:16:40.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:16:40.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:16:40.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:16:40.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:16:40.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:16:40.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.128.125:43165
16:16:40.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.128.125
16:16:40.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:16:40.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/krb5kdc/kudu_127.24.128.125.keytab
16:16:40.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.128.125
16:16:40.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:16:40.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:16:40.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:16:40.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.128.84:35767
16:16:40.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:16:40.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:16:40.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:16:40.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:16:40.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.128.126:46405,127.24.128.125:43165,127.24.128.124:43927
16:16:40.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:16:40.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38553/jwks.json
16:16:40.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:16:40.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:16:40 dist-test-slave-fnz7 krb5kdc[25101](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790439400, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.128.126@KRBTEST.COM for kudu/127.24.128.126@KRBTEST.COM
16:16:40.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:40.256134 25184 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.128.125" port: 43165 } has no permanent_uuid. Determining permanent_uuid...
16:16:40.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:40.258773 25133 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.24.128.125:43165: connect: Connection refused (error 111)
16:16:40.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:40.263342 25184 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.128.125:43165: Network error: Client connection negotiation failed: client connection to 127.24.128.125:43165: connect: Connection refused (error 111)
16:16:40.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:40.312880 25184 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.128.125" port: 43165 } attempt: 1
16:16:40.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:40.316498 25184 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.128.125:43165: Network error: Client connection negotiation failed: client connection to 127.24.128.125:43165: connect: Connection refused (error 111)
16:16:40.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:40.384928 25184 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.128.125" port: 43165 } attempt: 2
16:16:40.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:40.388643 25184 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.128.125:43165: Network error: Client connection negotiation failed: client connection to 127.24.128.125:43165: connect: Connection refused (error 111)
16:16:40.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:40.480115 25184 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.128.125" port: 43165 } attempt: 3
16:16:40.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:40.483740 25184 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.128.125:43165: Network error: Client connection negotiation failed: client connection to 127.24.128.125:43165: connect: Connection refused (error 111)
16:16:40.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:40.573541 25190 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:16:40.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:40.574200 25190 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:16:40.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:40.574790 25190 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:16:40.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:40.607987 25190 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
16:16:40.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:40.608253 25190 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:16:40.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:40.608526 25190 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:16:40.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:40.608722 25190 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:16:40.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:40.608911 25190 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:16:40.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:40.609095 25190 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:16:40.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:40.609360 25190 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38553/jwks.json
16:16:40.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:40.627214 25184 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.128.125" port: 43165 } attempt: 4
16:16:40.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:40.630988 25184 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.128.125:43165: Network error: Client connection negotiation failed: client connection to 127.24.128.125:43165: connect: Connection refused (error 111)
16:16:40.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:40.648130 25190 master_runner.cc:386] Master server non-default flags:
16:16:40.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:16:40.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.128.84:35767
16:16:40.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:16:40.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:16:40.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-1/data
16:16:40.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-1/wal
16:16:40.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
16:16:40.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:16:40.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.128.126:46405,127.24.128.125:43165,127.24.128.124:43927
16:16:40.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/krb5kdc/kudu_127.24.128.125.keytab
16:16:40.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.128.125
16:16:40.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:16:40.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:16:40.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:16:40.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.128.125:43165
16:16:40.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:16:40.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:16:40.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:16:40.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38553/jwks.json
16:16:40.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:16:40.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:16:40.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:16:40.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:16:40.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:16:40.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-1/data/info.pb
16:16:40.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.128.125
16:16:40.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:16:40.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:16:40.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:16:40.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:16:40.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:16:40.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:16:40.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-1/logs
16:16:40.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:16:40.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:16:40.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:16:40.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:16:40.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
16:16:40.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
16:16:40.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:16:40.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 14:44:58 UTC on ccycloud.quasar-hxpbms.root.comops.site
16:16:40.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4674114
16:16:40.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:16:40.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:40.649237 25190 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:16:40.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:40.650318 25190 file_cache.cc:493] Constructed file cache file cache with capacity 419430
16:16:40.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:40.669513 25200 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:16:40.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:40.930495 25184 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.128.125" port: 43165 } attempt: 5
16:16:40.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:40.935254 25184 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.128.125:43165: Network error: Client connection negotiation failed: client connection to 127.24.128.125:43165: connect: Connection refused (error 111)
16:16:41.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:41.482738 25184 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.128.125" port: 43165 } attempt: 6
16:16:41.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:41.489549 25184 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.128.125:43165: Network error: Client connection negotiation failed: client connection to 127.24.128.125:43165: connect: Connection refused (error 111)
16:16:41.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:40.669713 25198 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:16:41.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:40.670621 25197 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:16:41.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:41.907143 25199 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
16:16:41.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:41.907434 25190 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:16:41.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:41.912350 25190 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:16:41.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:41.915447 25190 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:16:41.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:41.916889 25190 hybrid_clock.cc:648] HybridClock initialized: now 1790439401916816 us; error 85 us; skew 500 ppm
16:16:41.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:41.918089 25190 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:16:41.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:16:41 dist-test-slave-fnz7 krb5kdc[25101](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790439401, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.128.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:16:41.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:41.931363 25190 init.cc:377] Logged in from keytab as kudu/127.24.128.125@KRBTEST.COM (short username kudu)
16:16:41.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:41.936812 25190 webserver.cc:460] Webserver started at http://127.24.128.125:40325/ using document root <none> and password file <none>
16:16:41.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:41.937732 25190 fs_manager.cc:359] Metadata directory not provided
16:16:41.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:41.937919 25190 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:16:41.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:41.938303 25190 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
16:16:41.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:41.943564 25190 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-1/data/instance:
16:16:41.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5440fed9b93c45c580cd974996c8187c"
16:16:41.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:16:41 on dist-test-slave-fnz7"
16:16:41.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:41.944733 25190 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-1/wal/instance:
16:16:41.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5440fed9b93c45c580cd974996c8187c"
16:16:41.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:16:41 on dist-test-slave-fnz7"
16:16:41.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:41.952507 25190 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.003s sys 0.004s
16:16:41.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:41.958626 25211 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:16:41.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:41.959949 25190 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
16:16:41.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:41.960294 25190 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-1/data,/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-1/wal
16:16:41.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5440fed9b93c45c580cd974996c8187c"
16:16:41.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:16:41 on dist-test-slave-fnz7"
16:16:41.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:41.960613 25190 fs_report.cc:383] FS layout report
16:16:41.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:16:41.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-1/wal
16:16:41.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-1/wal
16:16:41.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-1/data/data
16:16:41.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:16:41.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:16:41.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:16:41.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:16:41.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:16:41.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:16:41.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:16:41.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:16:41.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:16:41.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:16:41.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:16:41.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:41.995390 25190 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
16:16:42.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:42.001577 25190 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:16:42.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:42.002067 25190 kserver.cc:163] Server-wide thread pool size limit: 3276
16:16:42.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:42.084604 25190 rpc_server.cc:307] RPC server started. Bound to: 127.24.128.125:43165
16:16:42.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:42.084697 25263 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.24.128.125:43165 every 8 connection(s)
16:16:42.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:42.087411 25190 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-1/data/info.pb
16:16:42.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:42.091500 25089 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskkW_qFj/build/tsan/bin/kudu as pid 25190
16:16:42.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:42.092020 25089 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-1/wal/instance
16:16:42.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:42.093643 25264 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
16:16:42.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.24.128.124@KRBTEST.COM; defaulting to no policy
16:16:42.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:42.108512 25264 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.128.126" port: 46405 } has no permanent_uuid. Determining permanent_uuid...
16:16:42.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:16:42.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.24.128.124@KRBTEST.COM" created.
16:16:42.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:16:42 dist-test-slave-fnz7 krb5kdc[25101](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790439401, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.128.125@KRBTEST.COM for kudu/127.24.128.126@KRBTEST.COM
16:16:42.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:16:42.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.24.128.124 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/krb5kdc/kudu_127.24.128.124.keytab.
16:16:42.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.24.128.124 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/krb5kdc/kudu_127.24.128.124.keytab.
16:16:42.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:42.144835 25089 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskkW_qFj/build/tsan/bin/kudu
16:16:42.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskkW_qFj/build/tsan/bin/kudu
16:16:42.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-2/wal
16:16:42.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-2/data
16:16:42.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:16:42.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:16:42.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:16:42.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:16:42.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:16:42.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:16:42.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-2/logs
16:16:42.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-2/data/info.pb
16:16:42.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:16:42.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:16:42.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:16:42.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:16:42.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:16:42.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:16:42.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:16:42.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:16:42.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:16:42.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:16:42.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:16:42.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:16:42.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.128.124:43927
16:16:42.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.128.124
16:16:42.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:16:42.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/krb5kdc/kudu_127.24.128.124.keytab
16:16:42.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.128.124
16:16:42.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:16:42.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:16:42.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:16:42.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.128.84:35767
16:16:42.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:16:42.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:16:42.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:16:42.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:16:42.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.128.126:46405,127.24.128.125:43165,127.24.128.124:43927
16:16:42.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:16:42.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38553/jwks.json
16:16:42.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:16:42.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:42.153832 25264 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.128.125" port: 43165 } has no permanent_uuid. Determining permanent_uuid...
16:16:42.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:16:42 dist-test-slave-fnz7 krb5kdc[25101](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790439401, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.128.125@KRBTEST.COM for kudu/127.24.128.125@KRBTEST.COM
16:16:42.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:42.205936 25264 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.128.124" port: 43927 } has no permanent_uuid. Determining permanent_uuid...
16:16:42.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:42.208436 25213 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.24.128.124:43927: connect: Connection refused (error 111)
16:16:42.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:42.214377 25264 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.128.124:43927: Network error: Client connection negotiation failed: client connection to 127.24.128.124:43927: connect: Connection refused (error 111)
16:16:42.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:42.263973 25264 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.128.124" port: 43927 } attempt: 1
16:16:42.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:42.268188 25264 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.128.124:43927: Network error: Client connection negotiation failed: client connection to 127.24.128.124:43927: connect: Connection refused (error 111)
16:16:42.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:42.336728 25264 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.128.124" port: 43927 } attempt: 2
16:16:42.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:42.341183 25264 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.128.124:43927: Network error: Client connection negotiation failed: client connection to 127.24.128.124:43927: connect: Connection refused (error 111)
16:16:42.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:42.432646 25264 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.128.124" port: 43927 } attempt: 3
16:16:42.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:42.436982 25264 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.128.124:43927: Network error: Client connection negotiation failed: client connection to 127.24.128.124:43927: connect: Connection refused (error 111)
16:16:42.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:42.489949 25270 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:16:42.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:42.490643 25270 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:16:42.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:42.491432 25270 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:16:42.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:42.523676 25270 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
16:16:42.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:42.523952 25270 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:16:42.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:42.524358 25270 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:16:42.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:42.524677 25270 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:16:42.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:42.524994 25270 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:16:42.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:42.525306 25270 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:16:42.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:42.525538 25270 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38553/jwks.json
16:16:42.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:42.550184 25184 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.128.125" port: 43165 } attempt: 7
16:16:42.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:16:42 dist-test-slave-fnz7 krb5kdc[25101](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790439400, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.128.126@KRBTEST.COM for kudu/127.24.128.125@KRBTEST.COM
16:16:42.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:42.566385 25270 master_runner.cc:386] Master server non-default flags:
16:16:42.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:16:42.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.128.84:35767
16:16:42.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:16:42.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:16:42.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-2/data
16:16:42.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-2/wal
16:16:42.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
16:16:42.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:16:42.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.128.126:46405,127.24.128.125:43165,127.24.128.124:43927
16:16:42.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/krb5kdc/kudu_127.24.128.124.keytab
16:16:42.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.128.124
16:16:42.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:16:42.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:16:42.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:16:42.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.128.124:43927
16:16:42.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:16:42.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:16:42.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:16:42.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38553/jwks.json
16:16:42.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:16:42.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:16:42.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:16:42.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:16:42.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:16:42.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-2/data/info.pb
16:16:42.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.128.124
16:16:42.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:16:42.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:16:42.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:16:42.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:16:42.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:16:42.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:16:42.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-2/logs
16:16:42.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:16:42.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:16:42.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:16:42.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:16:42.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
16:16:42.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
16:16:42.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:16:42.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 14:44:58 UTC on ccycloud.quasar-hxpbms.root.comops.site
16:16:42.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4674114
16:16:42.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:16:42.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:42.567981 25270 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:16:42.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:42.569346 25270 file_cache.cc:493] Constructed file cache file cache with capacity 419430
16:16:42.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:42.577070 25184 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.128.124" port: 43927 } has no permanent_uuid. Determining permanent_uuid...
16:16:42.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:42.580461 25264 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.128.124" port: 43927 } attempt: 4
16:16:42.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:42.585393 25264 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.128.124:43927: Network error: Client connection negotiation failed: client connection to 127.24.128.124:43927: connect: Connection refused (error 111)
16:16:42.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:42.585599 25184 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.128.124:43927: Network error: Client connection negotiation failed: client connection to 127.24.128.124:43927: connect: Connection refused (error 111)
16:16:42.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:42.593811 25282 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:16:42.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:42.644205 25184 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.128.124" port: 43927 } attempt: 1
16:16:42.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:42.650125 25184 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.128.124:43927: Network error: Client connection negotiation failed: client connection to 127.24.128.124:43927: connect: Connection refused (error 111)
16:16:42.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:42.731765 25184 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.128.124" port: 43927 } attempt: 2
16:16:42.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:42.737120 25184 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.128.124:43927: Network error: Client connection negotiation failed: client connection to 127.24.128.124:43927: connect: Connection refused (error 111)
16:16:42.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:42.822546 25184 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.128.124" port: 43927 } attempt: 3
16:16:42.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:42.827572 25184 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.128.124:43927: Network error: Client connection negotiation failed: client connection to 127.24.128.124:43927: connect: Connection refused (error 111)
16:16:42.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:42.884830 25264 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.128.124" port: 43927 } attempt: 5
16:16:42.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:42.888605 25264 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.128.124:43927: Network error: Client connection negotiation failed: client connection to 127.24.128.124:43927: connect: Connection refused (error 111)
16:16:42.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:42.967998 25184 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.128.124" port: 43927 } attempt: 4
16:16:42.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:42.972862 25184 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.128.124:43927: Network error: Client connection negotiation failed: client connection to 127.24.128.124:43927: connect: Connection refused (error 111)
16:16:43.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:43.256264 25184 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.128.124" port: 43927 } attempt: 5
16:16:43.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:43.261706 25184 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.128.124:43927: Network error: Client connection negotiation failed: client connection to 127.24.128.124:43927: connect: Connection refused (error 111)
16:16:43.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:43.436285 25264 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.128.124" port: 43927 } attempt: 6
16:16:43.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:43.444489 25264 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.128.124:43927: Network error: Client connection negotiation failed: client connection to 127.24.128.124:43927: connect: Connection refused (error 111)
16:16:43.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:43.814303 25184 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.128.124" port: 43927 } attempt: 6
16:16:43.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:43.821633 25184 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.128.124:43927: Network error: Client connection negotiation failed: client connection to 127.24.128.124:43927: connect: Connection refused (error 111)
16:16:43.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:42.594132 25280 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:16:43.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:42.593832 25284 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:16:43.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:43.853462 25283 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
16:16:43.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:43.853583 25270 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:16:43.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:43.857998 25270 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:16:43.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:43.861254 25270 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:16:43.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:43.862735 25270 hybrid_clock.cc:648] HybridClock initialized: now 1790439403862681 us; error 54 us; skew 500 ppm
16:16:43.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:43.863705 25270 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:16:43.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:16:43 dist-test-slave-fnz7 krb5kdc[25101](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790439403, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.128.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:16:43.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:43.869645 25270 init.cc:377] Logged in from keytab as kudu/127.24.128.124@KRBTEST.COM (short username kudu)
16:16:43.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:43.874989 25270 webserver.cc:460] Webserver started at http://127.24.128.124:37655/ using document root <none> and password file <none>
16:16:43.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:43.876044 25270 fs_manager.cc:359] Metadata directory not provided
16:16:43.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:43.876305 25270 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:16:43.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:43.876889 25270 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
16:16:43.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:43.882089 25270 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-2/data/instance:
16:16:43.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "48372f8622dc4b11845f2254fcfffc3d"
16:16:43.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:16:43 on dist-test-slave-fnz7"
16:16:43.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:43.883345 25270 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-2/wal/instance:
16:16:43.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "48372f8622dc4b11845f2254fcfffc3d"
16:16:43.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:16:43 on dist-test-slave-fnz7"
16:16:43.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:43.890810 25270 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.002s sys 0.004s
16:16:43.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:43.896546 25296 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:16:43.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:43.897785 25270 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
16:16:43.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:43.898198 25270 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-2/data,/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-2/wal
16:16:43.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "48372f8622dc4b11845f2254fcfffc3d"
16:16:43.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:16:43 on dist-test-slave-fnz7"
16:16:43.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:43.898605 25270 fs_report.cc:383] FS layout report
16:16:43.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:16:43.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-2/wal
16:16:43.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-2/wal
16:16:43.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-2/data/data
16:16:43.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:16:43.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:16:43.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:16:43.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:16:43.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:16:43.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:16:43.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:16:43.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:16:43.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:16:43.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:16:43.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:16:43.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:43.941839 25270 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
16:16:43.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:43.947851 25270 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:16:43.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:43.948266 25270 kserver.cc:163] Server-wide thread pool size limit: 3276
16:16:44.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:44.028370 25270 rpc_server.cc:307] RPC server started. Bound to: 127.24.128.124:43927
16:16:44.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:44.028470 25348 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.24.128.124:43927 every 8 connection(s)
16:16:44.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:44.030843 25270 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-2/data/info.pb
16:16:44.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:44.034940 25089 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskkW_qFj/build/tsan/bin/kudu as pid 25270
16:16:44.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:44.035349 25089 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-2/wal/instance
16:16:44.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:44.036135 25349 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
16:16:44.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:44.048946 25349 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.128.126" port: 46405 } has no permanent_uuid. Determining permanent_uuid...
16:16:44.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:16:44 dist-test-slave-fnz7 krb5kdc[25101](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790439403, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.128.124@KRBTEST.COM for kudu/127.24.128.126@KRBTEST.COM
16:16:44.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:16:44 dist-test-slave-fnz7 krb5kdc[25101](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790439398, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.24.128.124@KRBTEST.COM
16:16:44.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:44.084749 25349 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.128.125" port: 43165 } has no permanent_uuid. Determining permanent_uuid...
16:16:44.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:16:44 dist-test-slave-fnz7 krb5kdc[25101](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790439403, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.128.124@KRBTEST.COM for kudu/127.24.128.125@KRBTEST.COM
16:16:44.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:16:44 dist-test-slave-fnz7 krb5kdc[25101](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790439398, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.24.128.125@KRBTEST.COM
16:16:44.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:44.118933 25349 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.128.124" port: 43927 } has no permanent_uuid. Determining permanent_uuid...
16:16:44.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:16:44 dist-test-slave-fnz7 krb5kdc[25101](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790439403, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.128.124@KRBTEST.COM for kudu/127.24.128.124@KRBTEST.COM
16:16:44.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:16:44 dist-test-slave-fnz7 krb5kdc[25101](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790439398, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.24.128.126@KRBTEST.COM
16:16:44.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:44.152007 25349 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d: Bootstrap starting.
16:16:44.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:44.157102 25089 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
16:16:44.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:44.158584 25349 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d: Neither blocks nor log segments found. Creating new log.
16:16:44.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:44.161065 25349 log.cc:824] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d: Log is configured to *not* fsync() on all Append() calls
16:16:44.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:44.166893 25349 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d: No bootstrap required, opened a new log
16:16:44.167 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:16:44.169 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
16:16:44.192 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
16:16:44.193 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
16:16:44.199 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.24.128.126"
port: 46405
}
bound_http_address {
host: "127.24.128.126"
port: 45197
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.24.128.125"
port: 43165
}
bound_http_address {
host: "127.24.128.125"
port: 40325
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.24.128.124"
port: 43927
}
bound_http_address {
host: "127.24.128.124"
port: 37655
}
}
}
16:16:44.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:44.198088 25349 raft_consensus.cc:348] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a14e6a5198641059abfde130f5ae6ff" member_type: VOTER last_known_addr { host: "127.24.128.126" port: 46405 } } peers { permanent_uuid: "5440fed9b93c45c580cd974996c8187c" member_type: VOTER last_known_addr { host: "127.24.128.125" port: 43165 } } peers { permanent_uuid: "48372f8622dc4b11845f2254fcfffc3d" member_type: VOTER last_known_addr { host: "127.24.128.124" port: 43927 } }
16:16:44.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:44.199128 25349 raft_consensus.cc:374] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
16:16:44.199 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
16:16:44.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:44.199462 25349 raft_consensus.cc:729] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 48372f8622dc4b11845f2254fcfffc3d, State: Initialized, Role: FOLLOWER
16:16:44.200 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:16:44.201 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
16:16:44.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:44.200428 25349 consensus_queue.cc:260] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a14e6a5198641059abfde130f5ae6ff" member_type: VOTER last_known_addr { host: "127.24.128.126" port: 46405 } } peers { permanent_uuid: "5440fed9b93c45c580cd974996c8187c" member_type: VOTER last_known_addr { host: "127.24.128.125" port: 43165 } } peers { permanent_uuid: "48372f8622dc4b11845f2254fcfffc3d" member_type: VOTER last_known_addr { host: "127.24.128.124" port: 43927 } }
16:16:44.203 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
16:16:44.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:44.208546 25358 sys_catalog.cc:455] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a14e6a5198641059abfde130f5ae6ff" member_type: VOTER last_known_addr { host: "127.24.128.126" port: 46405 } } peers { permanent_uuid: "5440fed9b93c45c580cd974996c8187c" member_type: VOTER last_known_addr { host: "127.24.128.125" port: 43165 } } peers { permanent_uuid: "48372f8622dc4b11845f2254fcfffc3d" member_type: VOTER last_known_addr { host: "127.24.128.124" port: 43927 } } }
16:16:44.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:44.209612 25358 sys_catalog.cc:458] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d [sys.catalog]: This master's current role is: FOLLOWER
16:16:44.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:44.210876 25349 sys_catalog.cc:564] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d [sys.catalog]: configured and running, proceeding with master startup.
16:16:44.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:16:44 dist-test-slave-fnz7 krb5kdc[25101](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790439404, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:16:44.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
16:16:44.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:44.244930 25371 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:16:44.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:44.245282 25371 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:16:44.249 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:16:44.251 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/krb5kdc/krb5cc
16:16:44.258 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
16:16:44.259 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/krb5kdc/krb5cc
16:16:44.261 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
16:16:44.265 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
16:16:44.267 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
16:16:44.267 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
16:16:44.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:44.239746 25270 master_runner.cc:186] Error getting master registration for 127.24.128.126:46405: OK, instance_id { permanent_uuid: "1a14e6a5198641059abfde130f5ae6ff" instance_seqno: 1790439400141467 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
16:16:44.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:44.276734 25270 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:16:44.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:16:44 dist-test-slave-fnz7 krb5kdc[25101](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790439404, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:16:44.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
16:16:44.301 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:16:44.301 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/krb5kdc/krb5cc
16:16:44.303 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
16:16:44.303 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.24.128.125:43165
16:16:44.305 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
16:16:44.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:44.305771 25089 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskkW_qFj/build/tsan/bin/kudu with pid 25190
16:16:44.333 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:16:44.333 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.24.128.126:46405
16:16:44.333 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
16:16:44.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:44.334053 25089 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskkW_qFj/build/tsan/bin/kudu with pid 25114
16:16:44.363 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:16:44.363 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.24.128.124:43927
16:16:44.364 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
16:16:44.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:44.364339 25089 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskkW_qFj/build/tsan/bin/kudu with pid 25270
16:16:44.382 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:16:44.383 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.24.128.125:43165
16:16:44.385 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
16:16:44.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:44.387610 25089 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskkW_qFj/build/tsan/bin/kudu
16:16:44.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskkW_qFj/build/tsan/bin/kudu
16:16:44.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-1/wal
16:16:44.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-1/data
16:16:44.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:16:44.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:16:44.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:16:44.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:16:44.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:16:44.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:16:44.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-1/logs
16:16:44.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-1/data/info.pb
16:16:44.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:16:44.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:16:44.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:16:44.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:16:44.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:16:44.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:16:44.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:16:44.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:16:44.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:16:44.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:16:44.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:16:44.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:16:44.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.128.125:43165
16:16:44.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.128.125
16:16:44.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40325
16:16:44.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/krb5kdc/kudu_127.24.128.125.keytab
16:16:44.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.128.125
16:16:44.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:16:44.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:16:44.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:16:44.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.128.84:35767
16:16:44.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:16:44.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:16:44.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:16:44.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:16:44.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.128.126:46405,127.24.128.125:43165,127.24.128.124:43927
16:16:44.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:16:44.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38553/jwks.json
16:16:44.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:16:44.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:44.727891 25375 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:16:44.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:44.728472 25375 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:16:44.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:44.729075 25375 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:16:44.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:44.763087 25375 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
16:16:44.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:44.763447 25375 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:16:44.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:44.763749 25375 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:16:44.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:44.763955 25375 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:16:44.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:44.764168 25375 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:16:44.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:44.764371 25375 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:16:44.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:44.764516 25375 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38553/jwks.json
16:16:44.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:44.807539 25375 master_runner.cc:386] Master server non-default flags:
16:16:44.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:16:44.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.128.84:35767
16:16:44.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:16:44.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:16:44.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-1/data
16:16:44.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-1/wal
16:16:44.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
16:16:44.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:16:44.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.128.126:46405,127.24.128.125:43165,127.24.128.124:43927
16:16:44.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/krb5kdc/kudu_127.24.128.125.keytab
16:16:44.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.128.125
16:16:44.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:16:44.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:16:44.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:16:44.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.128.125:43165
16:16:44.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:16:44.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:16:44.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:16:44.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38553/jwks.json
16:16:44.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:16:44.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:16:44.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:16:44.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:16:44.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:16:44.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-1/data/info.pb
16:16:44.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.128.125
16:16:44.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40325
16:16:44.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:16:44.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:16:44.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:16:44.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:16:44.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:16:44.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-1/logs
16:16:44.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:16:44.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:16:44.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:16:44.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:16:44.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
16:16:44.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
16:16:44.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:16:44.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 14:44:58 UTC on ccycloud.quasar-hxpbms.root.comops.site
16:16:44.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4674114
16:16:44.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:16:44.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:44.809115 25375 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:16:44.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:44.810508 25375 file_cache.cc:493] Constructed file cache file cache with capacity 419430
16:16:44.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:44.829370 25383 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:16:46.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:44.829382 25382 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:16:46.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:44.829391 25385 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:16:46.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:44.831423 25375 server_base.cc:1034] running on GCE node
16:16:46.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:46.076869 25375 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:16:46.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:46.079489 25375 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:16:46.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:46.080915 25375 hybrid_clock.cc:648] HybridClock initialized: now 1790439406080882 us; error 53 us; skew 500 ppm
16:16:46.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:46.081715 25375 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:16:46.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:16:46 dist-test-slave-fnz7 krb5kdc[25101](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790439406, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.128.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:16:46.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:46.087954 25375 init.cc:377] Logged in from keytab as kudu/127.24.128.125@KRBTEST.COM (short username kudu)
16:16:46.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:46.093441 25375 webserver.cc:460] Webserver started at http://127.24.128.125:40325/ using document root <none> and password file <none>
16:16:46.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:46.094739 25375 fs_manager.cc:359] Metadata directory not provided
16:16:46.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:46.095046 25375 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:16:46.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:46.103711 25375 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.003s sys 0.004s
16:16:46.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:46.108680 25395 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:16:46.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:46.110147 25375 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
16:16:46.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:46.110461 25375 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-1/data,/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-1/wal
16:16:46.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5440fed9b93c45c580cd974996c8187c"
16:16:46.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:16:41 on dist-test-slave-fnz7"
16:16:46.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:46.112910 25375 fs_report.cc:383] FS layout report
16:16:46.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:16:46.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-1/wal
16:16:46.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-1/wal
16:16:46.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-1/data/data
16:16:46.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:16:46.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:16:46.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:16:46.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:16:46.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:16:46.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:16:46.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:16:46.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:16:46.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:16:46.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:16:46.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:16:46.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:46.167999 25375 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
16:16:46.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:46.175803 25375 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:16:46.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:46.176347 25375 kserver.cc:163] Server-wide thread pool size limit: 3276
16:16:46.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:46.257189 25375 rpc_server.cc:307] RPC server started. Bound to: 127.24.128.125:43165
16:16:46.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:46.257320 25446 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.24.128.125:43165 every 8 connection(s)
16:16:46.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:46.260293 25375 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-1/data/info.pb
16:16:46.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:46.268844 25089 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskkW_qFj/build/tsan/bin/kudu as pid 25375
16:16:46.269 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:16:46.269 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.24.128.126:46405
16:16:46.270 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
16:16:46.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:46.271719 25089 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskkW_qFj/build/tsan/bin/kudu
16:16:46.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskkW_qFj/build/tsan/bin/kudu
16:16:46.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-0/wal
16:16:46.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-0/data
16:16:46.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:16:46.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:16:46.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:16:46.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:16:46.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:16:46.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:16:46.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-0/logs
16:16:46.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-0/data/info.pb
16:16:46.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:16:46.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:16:46.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:16:46.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:16:46.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:16:46.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:16:46.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:16:46.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:16:46.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:16:46.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:16:46.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:16:46.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:16:46.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.128.126:46405
16:16:46.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.128.126
16:16:46.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45197
16:16:46.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/krb5kdc/kudu_127.24.128.126.keytab
16:16:46.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.128.126
16:16:46.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:16:46.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:16:46.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:16:46.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.128.84:35767
16:16:46.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:16:46.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:16:46.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:16:46.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:16:46.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.128.126:46405,127.24.128.125:43165,127.24.128.124:43927
16:16:46.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:16:46.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38553/jwks.json
16:16:46.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:16:46.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:46.274010 25447 sys_catalog.cc:263] Verifying existing consensus state
16:16:46.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260926 16:16:46.275127 25447 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
16:16:46.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:46.275727 25375 master.cc:537] Master@127.24.128.125:43165 shutting down...
16:16:46.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:46.342900 25375 master.cc:552] Master@127.24.128.125:43165 shutdown complete.
16:16:46.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
16:16:46.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:46.609102 25449 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:16:46.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:46.609834 25449 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:16:46.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:46.610584 25449 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:16:46.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:46.642656 25449 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
16:16:46.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:46.642968 25449 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:16:46.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:46.643433 25449 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:16:46.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:46.643766 25449 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:16:46.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:46.644078 25449 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:16:46.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:46.644409 25449 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:16:46.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:46.644655 25449 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38553/jwks.json
16:16:46.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:46.691874 25449 master_runner.cc:386] Master server non-default flags:
16:16:46.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:16:46.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.128.84:35767
16:16:46.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:16:46.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:16:46.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-0/data
16:16:46.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-0/wal
16:16:46.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
16:16:46.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:16:46.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.128.126:46405,127.24.128.125:43165,127.24.128.124:43927
16:16:46.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/krb5kdc/kudu_127.24.128.126.keytab
16:16:46.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.128.126
16:16:46.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:16:46.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:16:46.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:16:46.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.128.126:46405
16:16:46.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:16:46.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:16:46.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:16:46.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38553/jwks.json
16:16:46.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:16:46.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:16:46.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:16:46.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:16:46.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:16:46.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-0/data/info.pb
16:16:46.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.128.126
16:16:46.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45197
16:16:46.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:16:46.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:16:46.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:16:46.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:16:46.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:16:46.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-0/logs
16:16:46.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:16:46.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:16:46.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:16:46.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:16:46.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
16:16:46.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
16:16:46.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:16:46.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 14:44:58 UTC on ccycloud.quasar-hxpbms.root.comops.site
16:16:46.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4674114
16:16:46.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:16:46.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:46.693151 25449 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:16:46.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:46.694480 25449 file_cache.cc:493] Constructed file cache file cache with capacity 419430
16:16:46.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:46.714232 25458 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:16:47.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:46.714401 25460 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:16:47.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:46.715103 25457 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:16:47.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:46.715957 25449 server_base.cc:1034] running on GCE node
16:16:47.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:47.970084 25449 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:16:47.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:47.972895 25449 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:16:47.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:47.974318 25449 hybrid_clock.cc:648] HybridClock initialized: now 1790439407974286 us; error 41 us; skew 500 ppm
16:16:47.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:47.975208 25449 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:16:47.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:16:47 dist-test-slave-fnz7 krb5kdc[25101](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790439407, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.128.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:16:47.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:47.986127 25449 init.cc:377] Logged in from keytab as kudu/127.24.128.126@KRBTEST.COM (short username kudu)
16:16:47.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:47.991890 25449 webserver.cc:460] Webserver started at http://127.24.128.126:45197/ using document root <none> and password file <none>
16:16:47.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:47.992946 25449 fs_manager.cc:359] Metadata directory not provided
16:16:47.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:47.993216 25449 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:16:48.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:48.001652 25449 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.001s sys 0.004s
16:16:48.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:48.006567 25469 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:16:48.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:48.007831 25449 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.001s sys 0.002s
16:16:48.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:48.008199 25449 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-0/data,/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-0/wal
16:16:48.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1a14e6a5198641059abfde130f5ae6ff"
16:16:48.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:16:40 on dist-test-slave-fnz7"
16:16:48.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:48.010934 25449 fs_report.cc:383] FS layout report
16:16:48.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:16:48.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-0/wal
16:16:48.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-0/wal
16:16:48.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-0/data/data
16:16:48.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:16:48.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:16:48.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:16:48.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:16:48.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:16:48.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:16:48.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:16:48.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:16:48.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:16:48.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:16:48.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:16:48.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:48.048183 25449 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
16:16:48.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:48.054608 25449 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:16:48.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:48.055240 25449 kserver.cc:163] Server-wide thread pool size limit: 3276
16:16:48.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:48.131240 25449 rpc_server.cc:307] RPC server started. Bound to: 127.24.128.126:46405
16:16:48.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:48.131501 25520 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.24.128.126:46405 every 8 connection(s)
16:16:48.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:48.134696 25449 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-0/data/info.pb
16:16:48.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:48.141446 25089 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskkW_qFj/build/tsan/bin/kudu as pid 25449
16:16:48.142 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:16:48.143 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.24.128.124:43927
16:16:48.143 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
16:16:48.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:48.145113 25089 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskkW_qFj/build/tsan/bin/kudu
16:16:48.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskkW_qFj/build/tsan/bin/kudu
16:16:48.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-2/wal
16:16:48.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-2/data
16:16:48.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:16:48.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:16:48.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:16:48.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:16:48.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:16:48.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:16:48.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-2/logs
16:16:48.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-2/data/info.pb
16:16:48.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:16:48.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:16:48.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:16:48.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:16:48.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:16:48.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:16:48.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:16:48.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:16:48.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:16:48.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:16:48.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:16:48.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:16:48.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.128.124:43927
16:16:48.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.128.124
16:16:48.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37655
16:16:48.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/krb5kdc/kudu_127.24.128.124.keytab
16:16:48.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.128.124
16:16:48.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:16:48.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:16:48.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:16:48.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.128.84:35767
16:16:48.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:16:48.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:16:48.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:16:48.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:16:48.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.128.126:46405,127.24.128.125:43165,127.24.128.124:43927
16:16:48.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:16:48.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38553/jwks.json
16:16:48.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:16:48.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:48.146445 25521 sys_catalog.cc:263] Verifying existing consensus state
16:16:48.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260926 16:16:48.147424 25521 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
16:16:48.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:48.147863 25449 master.cc:537] Master@127.24.128.126:46405 shutting down...
16:16:48.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:48.219646 25449 master.cc:552] Master@127.24.128.126:46405 shutdown complete.
16:16:48.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
16:16:48.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:48.475389 25523 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:16:48.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:48.476080 25523 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:16:48.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:48.476827 25523 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:16:48.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:48.509814 25523 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
16:16:48.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:48.510180 25523 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:16:48.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:48.510586 25523 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:16:48.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:48.510907 25523 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:16:48.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:48.511221 25523 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:16:48.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:48.511612 25523 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:16:48.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:48.511839 25523 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38553/jwks.json
16:16:48.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:48.549228 25523 master_runner.cc:386] Master server non-default flags:
16:16:48.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:16:48.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.128.84:35767
16:16:48.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:16:48.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:16:48.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-2/data
16:16:48.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-2/wal
16:16:48.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
16:16:48.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:16:48.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.128.126:46405,127.24.128.125:43165,127.24.128.124:43927
16:16:48.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/krb5kdc/kudu_127.24.128.124.keytab
16:16:48.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.128.124
16:16:48.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:16:48.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:16:48.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:16:48.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.128.124:43927
16:16:48.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:16:48.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:16:48.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:16:48.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38553/jwks.json
16:16:48.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:16:48.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:16:48.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:16:48.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:16:48.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:16:48.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-2/data/info.pb
16:16:48.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.128.124
16:16:48.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37655
16:16:48.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:16:48.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:16:48.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:16:48.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:16:48.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:16:48.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-2/logs
16:16:48.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:16:48.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:16:48.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:16:48.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:16:48.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
16:16:48.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
16:16:48.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:16:48.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 14:44:58 UTC on ccycloud.quasar-hxpbms.root.comops.site
16:16:48.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4674114
16:16:48.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:16:48.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:48.550789 25523 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:16:48.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:48.551960 25523 file_cache.cc:493] Constructed file cache file cache with capacity 419430
16:16:48.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:48.574401 25531 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:16:49.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:48.575613 25530 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:16:49.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:48.574527 25533 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:16:49.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:49.823846 25532 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
16:16:49.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:49.824074 25523 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:16:49.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:49.829037 25523 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:16:49.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:49.831794 25523 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:16:49.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:49.833240 25523 hybrid_clock.cc:648] HybridClock initialized: now 1790439409833177 us; error 58 us; skew 500 ppm
16:16:49.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:49.834102 25523 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:16:49.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:16:49 dist-test-slave-fnz7 krb5kdc[25101](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790439409, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.128.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:16:49.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:49.840241 25523 init.cc:377] Logged in from keytab as kudu/127.24.128.124@KRBTEST.COM (short username kudu)
16:16:49.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:49.846518 25523 webserver.cc:460] Webserver started at http://127.24.128.124:37655/ using document root <none> and password file <none>
16:16:49.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:49.847700 25523 fs_manager.cc:359] Metadata directory not provided
16:16:49.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:49.847980 25523 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:16:49.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:49.856208 25523 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.007s sys 0.000s
16:16:49.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:49.860895 25541 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:16:49.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:49.861987 25523 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
16:16:49.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:49.862368 25523 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-2/data,/tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-2/wal
16:16:49.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "48372f8622dc4b11845f2254fcfffc3d"
16:16:49.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:16:43 on dist-test-slave-fnz7"
16:16:49.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:49.864974 25523 fs_report.cc:383] FS layout report
16:16:49.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:16:49.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-2/wal
16:16:49.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-2/wal
16:16:49.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-2/data/data
16:16:49.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:16:49.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:16:49.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:16:49.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:16:49.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:16:49.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:16:49.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:16:49.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:16:49.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:16:49.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:16:49.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:16:49.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:49.907536 25523 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
16:16:49.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:49.914364 25523 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:16:49.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:49.914995 25523 kserver.cc:163] Server-wide thread pool size limit: 3276
16:16:49.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:49.989832 25523 rpc_server.cc:307] RPC server started. Bound to: 127.24.128.124:43927
16:16:49.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:49.989933 25592 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.24.128.124:43927 every 8 connection(s)
16:16:49.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:49.992842 25523 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskkW_qFj/test-tmp/mini-kudu-cluster3643366322984361479/master-2/data/info.pb
16:16:50.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:50.000725 25089 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskkW_qFj/build/tsan/bin/kudu as pid 25523
16:16:50.001 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:16:50.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:50.004563 25593 sys_catalog.cc:263] Verifying existing consensus state
16:16:50.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:50.011071 25593 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d: Bootstrap starting.
16:16:50.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:50.048822 25593 log.cc:824] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d: Log is configured to *not* fsync() on all Append() calls
16:16:50.050 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.24.128.125:43165(127.24.128.125:43165)] connecting to peer
16:16:50.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:50.054373 25593 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
16:16:50.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:50.055356 25593 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d: Bootstrap complete.
16:16:50.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:50.092389 25593 raft_consensus.cc:348] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a14e6a5198641059abfde130f5ae6ff" member_type: VOTER last_known_addr { host: "127.24.128.126" port: 46405 } } peers { permanent_uuid: "5440fed9b93c45c580cd974996c8187c" member_type: VOTER last_known_addr { host: "127.24.128.125" port: 43165 } } peers { permanent_uuid: "48372f8622dc4b11845f2254fcfffc3d" member_type: VOTER last_known_addr { host: "127.24.128.124" port: 43927 } }
16:16:50.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:50.095948 25593 raft_consensus.cc:374] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
16:16:50.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:50.096276 25593 raft_consensus.cc:729] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 48372f8622dc4b11845f2254fcfffc3d, State: Initialized, Role: FOLLOWER
16:16:50.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:50.097277 25593 consensus_queue.cc:260] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a14e6a5198641059abfde130f5ae6ff" member_type: VOTER last_known_addr { host: "127.24.128.126" port: 46405 } } peers { permanent_uuid: "5440fed9b93c45c580cd974996c8187c" member_type: VOTER last_known_addr { host: "127.24.128.125" port: 43165 } } peers { permanent_uuid: "48372f8622dc4b11845f2254fcfffc3d" member_type: VOTER last_known_addr { host: "127.24.128.124" port: 43927 } }
16:16:50.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:50.100512 25597 sys_catalog.cc:455] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a14e6a5198641059abfde130f5ae6ff" member_type: VOTER last_known_addr { host: "127.24.128.126" port: 46405 } } peers { permanent_uuid: "5440fed9b93c45c580cd974996c8187c" member_type: VOTER last_known_addr { host: "127.24.128.125" port: 43165 } } peers { permanent_uuid: "48372f8622dc4b11845f2254fcfffc3d" member_type: VOTER last_known_addr { host: "127.24.128.124" port: 43927 } } }
16:16:50.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:50.101686 25597 sys_catalog.cc:458] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d [sys.catalog]: This master's current role is: FOLLOWER
16:16:50.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:50.102229 25593 sys_catalog.cc:564] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d [sys.catalog]: configured and running, proceeding with master startup.
16:16:50.112 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.24.128.126:46405(127.24.128.126:46405)] connecting to peer
16:16:50.113 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.24.128.124:43927(127.24.128.124:43927)] connecting to peer
16:16:50.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:50.174425 25547 tablet.cc:2369] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d: Can't schedule compaction. Clean time has not been advanced past its initial value.
16:16:50.183 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:16:50.184 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.128.125:43165(127.24.128.125:43165)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:16:50.186 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.128.125:43165: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:16:50.187 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.24.128.124:43927(127.24.128.124:43927)] Successfully connected to peer
16:16:50.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:50.189489 25610 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:16:50.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:50.189844 25610 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:16:50.195 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:16:50.196 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.128.126:46405(127.24.128.126:46405)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:16:50.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:50.195998 25597 raft_consensus.cc:482] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
16:16:50.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:50.196528 25597 raft_consensus.cc:504] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a14e6a5198641059abfde130f5ae6ff" member_type: VOTER last_known_addr { host: "127.24.128.126" port: 46405 } } peers { permanent_uuid: "5440fed9b93c45c580cd974996c8187c" member_type: VOTER last_known_addr { host: "127.24.128.125" port: 43165 } } peers { permanent_uuid: "48372f8622dc4b11845f2254fcfffc3d" member_type: VOTER last_known_addr { host: "127.24.128.124" port: 43927 } }
16:16:50.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:50.200949 25543 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.24.128.126:46405: connect: Connection refused (error 111)
16:16:50.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:50.202657 25597 leader_election.cc:288] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1a14e6a5198641059abfde130f5ae6ff (127.24.128.126:46405), 5440fed9b93c45c580cd974996c8187c (127.24.128.125:43165)
16:16:50.203 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.128.126:46405: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:16:50.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:50.218178 25542 leader_election.cc:334] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 5440fed9b93c45c580cd974996c8187c (127.24.128.125:43165): Network error: Client connection negotiation failed: client connection to 127.24.128.125:43165: connect: Connection refused (error 111)
16:16:50.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:50.224150 25543 leader_election.cc:334] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 1a14e6a5198641059abfde130f5ae6ff (127.24.128.126:46405): Network error: Client connection negotiation failed: client connection to 127.24.128.126:46405: connect: Connection refused (error 111)
16:16:50.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:50.224932 25543 leader_election.cc:302] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 48372f8622dc4b11845f2254fcfffc3d; no voters: 1a14e6a5198641059abfde130f5ae6ff, 5440fed9b93c45c580cd974996c8187c
16:16:50.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:50.226238 25597 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
16:16:50.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:50.157198 25523 master_runner.cc:186] Error getting master registration for 127.24.128.126:46405: Network error: Client connection negotiation failed: client connection to 127.24.128.126:46405: connect: Connection refused (error 111),
16:16:50.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:50.229004 25523 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:16:50.302 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
16:16:50.316 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.24.128.124
16:16:50.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:16:50 dist-test-slave-fnz7 krb5kdc[25101](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790439404, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.24.128.124@KRBTEST.COM
16:16:50.599 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x4efbca9f, L:/127.0.0.1:40516 - R:/127.24.128.124:43927] using SASL/GSSAPI
16:16:50.621 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927; will retry
16:16:50.628 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.128.125:43165(127.24.128.125:43165)] connecting to peer
16:16:50.630 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.128.126:46405(127.24.128.126:46405)] connecting to peer
16:16:50.634 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:16:50.634 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.128.125:43165(127.24.128.125:43165)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:16:50.635 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.128.125:43165: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:16:50.638 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:16:50.638 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.128.126:46405(127.24.128.126:46405)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:16:50.639 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.128.126:46405: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:16:50.649 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927; will retry
16:16:50.668 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.128.125:43165(127.24.128.125:43165)] connecting to peer
16:16:50.669 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.128.126:46405(127.24.128.126:46405)] connecting to peer
16:16:50.670 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:16:50.671 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.128.125:43165(127.24.128.125:43165)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:16:50.671 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.128.125:43165: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:16:50.677 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:16:50.677 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.128.126:46405(127.24.128.126:46405)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:16:50.678 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.128.126:46405: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:16:50.679 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927; will retry
16:16:50.687 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.128.125:43165(127.24.128.125:43165)] connecting to peer
16:16:50.688 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.128.126:46405(127.24.128.126:46405)] connecting to peer
16:16:50.690 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:16:50.690 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.128.125:43165(127.24.128.125:43165)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:16:50.691 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.128.125:43165: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:16:50.693 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:16:50.693 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.128.126:46405(127.24.128.126:46405)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:16:50.693 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.128.126:46405: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:16:50.695 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927; will retry
16:16:50.707 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.128.125:43165(127.24.128.125:43165)] connecting to peer
16:16:50.709 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.128.126:46405(127.24.128.126:46405)] connecting to peer
16:16:50.710 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:16:50.710 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.128.125:43165(127.24.128.125:43165)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:16:50.711 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.128.125:43165: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:16:50.716 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:16:50.716 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.128.126:46405(127.24.128.126:46405)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:16:50.717 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.128.126:46405: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:16:50.718 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927; will retry
16:16:50.748 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.128.125:43165(127.24.128.125:43165)] connecting to peer
16:16:50.749 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.128.126:46405(127.24.128.126:46405)] connecting to peer
16:16:50.751 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:16:50.752 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.128.125:43165(127.24.128.125:43165)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:16:50.752 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.128.125:43165: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:16:50.753 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:16:50.754 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.128.126:46405(127.24.128.126:46405)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:16:50.754 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.128.126:46405: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:16:50.756 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927; will retry
16:16:50.787 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.128.125:43165(127.24.128.125:43165)] connecting to peer
16:16:50.789 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.128.126:46405(127.24.128.126:46405)] connecting to peer
16:16:50.794 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:16:50.794 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.128.125:43165(127.24.128.125:43165)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:16:50.796 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.128.125:43165: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:16:50.799 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:16:50.799 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.128.126:46405(127.24.128.126:46405)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:16:50.801 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.128.126:46405: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:16:50.801 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927; will retry
16:16:50.868 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.128.125:43165(127.24.128.125:43165)] connecting to peer
16:16:50.870 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.128.126:46405(127.24.128.126:46405)] connecting to peer
16:16:50.871 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:16:50.872 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.128.125:43165(127.24.128.125:43165)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:16:50.872 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.128.125:43165: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:16:50.873 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:16:50.874 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.128.126:46405(127.24.128.126:46405)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:16:50.874 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.128.126:46405: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:16:50.876 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927; will retry
16:16:51.087 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.128.125:43165(127.24.128.125:43165)] connecting to peer
16:16:51.090 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.128.126:46405(127.24.128.126:46405)] connecting to peer
16:16:51.090 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:16:51.090 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.128.125:43165(127.24.128.125:43165)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:16:51.095 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.128.125:43165: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:16:51.099 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:16:51.099 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.128.126:46405(127.24.128.126:46405)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:16:51.100 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.128.126:46405: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:16:51.102 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927; will retry
16:16:51.187 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.128.125:43165(127.24.128.125:43165)] connecting to peer
16:16:51.188 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.128.126:46405(127.24.128.126:46405)] connecting to peer
16:16:51.190 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:16:51.190 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.128.125:43165(127.24.128.125:43165)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:16:51.192 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.128.125:43165: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:16:51.193 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:16:51.193 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.128.126:46405(127.24.128.126:46405)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:16:51.194 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.128.126:46405: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:16:51.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:51.195955 25610 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:16:51.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:51.196300 25610 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:16:51.196 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927; will retry
16:16:51.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:51.826666 25618 raft_consensus.cc:482] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
16:16:51.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:51.827329 25618 raft_consensus.cc:504] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a14e6a5198641059abfde130f5ae6ff" member_type: VOTER last_known_addr { host: "127.24.128.126" port: 46405 } } peers { permanent_uuid: "5440fed9b93c45c580cd974996c8187c" member_type: VOTER last_known_addr { host: "127.24.128.125" port: 43165 } } peers { permanent_uuid: "48372f8622dc4b11845f2254fcfffc3d" member_type: VOTER last_known_addr { host: "127.24.128.124" port: 43927 } }
16:16:51.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:51.829063 25618 leader_election.cc:288] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1a14e6a5198641059abfde130f5ae6ff (127.24.128.126:46405), 5440fed9b93c45c580cd974996c8187c (127.24.128.125:43165)
16:16:51.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:51.835845 25542 leader_election.cc:334] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 5440fed9b93c45c580cd974996c8187c (127.24.128.125:43165): Network error: Client connection negotiation failed: client connection to 127.24.128.125:43165: connect: Connection refused (error 111)
16:16:51.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:51.836589 25543 leader_election.cc:334] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 1a14e6a5198641059abfde130f5ae6ff (127.24.128.126:46405): Network error: Client connection negotiation failed: client connection to 127.24.128.126:46405: connect: Connection refused (error 111)
16:16:51.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:51.836985 25543 leader_election.cc:302] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 48372f8622dc4b11845f2254fcfffc3d; no voters: 1a14e6a5198641059abfde130f5ae6ff, 5440fed9b93c45c580cd974996c8187c
16:16:51.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:51.837658 25618 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
16:16:51.947 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.128.125:43165(127.24.128.125:43165)] connecting to peer
16:16:51.949 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.128.126:46405(127.24.128.126:46405)] connecting to peer
16:16:51.950 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:16:51.950 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.128.125:43165(127.24.128.125:43165)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:16:51.950 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.128.125:43165: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:16:51.951 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:16:51.951 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.128.126:46405(127.24.128.126:46405)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:16:51.951 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.128.126:46405: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:16:51.954 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927; will retry
16:16:52.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:52.199847 25610 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:16:52.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:52.200088 25610 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:16:52.588 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.128.125:43165(127.24.128.125:43165)] connecting to peer
16:16:52.589 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.128.126:46405(127.24.128.126:46405)] connecting to peer
16:16:52.591 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:16:52.591 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.128.125:43165(127.24.128.125:43165)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:16:52.591 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.128.125:43165: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:16:52.592 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:16:52.592 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.128.126:46405(127.24.128.126:46405)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:16:52.593 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.128.126:46405: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:16:52.595 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927; will retry
16:16:53.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:53.203008 25610 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:16:53.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:53.203269 25610 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:16:53.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:53.704208 25624 raft_consensus.cc:482] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
16:16:53.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:53.704624 25624 raft_consensus.cc:504] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a14e6a5198641059abfde130f5ae6ff" member_type: VOTER last_known_addr { host: "127.24.128.126" port: 46405 } } peers { permanent_uuid: "5440fed9b93c45c580cd974996c8187c" member_type: VOTER last_known_addr { host: "127.24.128.125" port: 43165 } } peers { permanent_uuid: "48372f8622dc4b11845f2254fcfffc3d" member_type: VOTER last_known_addr { host: "127.24.128.124" port: 43927 } }
16:16:53.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:53.705983 25624 leader_election.cc:288] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1a14e6a5198641059abfde130f5ae6ff (127.24.128.126:46405), 5440fed9b93c45c580cd974996c8187c (127.24.128.125:43165)
16:16:53.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:53.712109 25543 leader_election.cc:334] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 1a14e6a5198641059abfde130f5ae6ff (127.24.128.126:46405): Network error: Client connection negotiation failed: client connection to 127.24.128.126:46405: connect: Connection refused (error 111)
16:16:53.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:53.712635 25542 leader_election.cc:334] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 5440fed9b93c45c580cd974996c8187c (127.24.128.125:43165): Network error: Client connection negotiation failed: client connection to 127.24.128.125:43165: connect: Connection refused (error 111)
16:16:53.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:53.713055 25542 leader_election.cc:302] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 48372f8622dc4b11845f2254fcfffc3d; no voters: 1a14e6a5198641059abfde130f5ae6ff, 5440fed9b93c45c580cd974996c8187c
16:16:53.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:53.713791 25624 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
16:16:54.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:54.206123 25610 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:16:54.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:54.206473 25610 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:16:54.407 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.128.125:43165(127.24.128.125:43165)] connecting to peer
16:16:54.408 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.128.126:46405(127.24.128.126:46405)] connecting to peer
16:16:54.409 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:16:54.410 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.128.125:43165(127.24.128.125:43165)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:16:54.410 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.128.125:43165: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:16:54.411 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:16:54.412 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.128.126:46405(127.24.128.126:46405)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:16:54.412 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.128.126:46405: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:16:54.414 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927; will retry
16:16:54.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:54.934917 25074 debug-util.cc:398] Leaking SignalData structure 0x7b08000be160 after lost signal to thread 24949
16:16:54.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:54.936168 25074 debug-util.cc:398] Leaking SignalData structure 0x7b08000bb940 after lost signal to thread 25077
16:16:55.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:55.209656 25610 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:16:55.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:55.209932 25610 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:16:55.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:55.232257 25523 master_runner.cc:186] Error getting master registration for 127.24.128.126:46405: Network error: Client connection negotiation failed: client connection to 127.24.128.126:46405: connect: Connection refused (error 111),
16:16:55.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:55.232657 25523 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:16:56.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:56.212997 25610 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:16:56.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:56.213315 25610 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:16:57.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:57.216506 25610 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:16:57.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:57.216866 25610 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:16:57.607 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.128.125:43165(127.24.128.125:43165)] connecting to peer
16:16:57.608 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.128.126:46405(127.24.128.126:46405)] connecting to peer
16:16:57.610 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:16:57.610 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.128.125:43165(127.24.128.125:43165)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:16:57.610 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.128.125:43165: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:16:57.611 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:16:57.611 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.128.126:46405(127.24.128.126:46405)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:16:57.611 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.128.126:46405: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:16:57.613 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927; will retry
16:16:57.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:57.623492 25074 debug-util.cc:398] Leaking SignalData structure 0x7b08000af180 after lost signal to thread 24949
16:16:57.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:57.624187 25074 debug-util.cc:398] Leaking SignalData structure 0x7b08000ba020 after lost signal to thread 25077
16:16:58.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:58.219851 25610 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:16:58.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:58.220206 25610 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:16:58.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:58.688006 25635 raft_consensus.cc:482] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
16:16:58.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:58.688635 25635 raft_consensus.cc:504] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a14e6a5198641059abfde130f5ae6ff" member_type: VOTER last_known_addr { host: "127.24.128.126" port: 46405 } } peers { permanent_uuid: "5440fed9b93c45c580cd974996c8187c" member_type: VOTER last_known_addr { host: "127.24.128.125" port: 43165 } } peers { permanent_uuid: "48372f8622dc4b11845f2254fcfffc3d" member_type: VOTER last_known_addr { host: "127.24.128.124" port: 43927 } }
16:16:58.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:58.690681 25635 leader_election.cc:288] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1a14e6a5198641059abfde130f5ae6ff (127.24.128.126:46405), 5440fed9b93c45c580cd974996c8187c (127.24.128.125:43165)
16:16:58.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:58.693392 25543 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.24.128.126:46405: connect: Connection refused (error 111) [suppressed 5 similar messages]
16:16:58.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:58.696442 25543 leader_election.cc:334] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 1a14e6a5198641059abfde130f5ae6ff (127.24.128.126:46405): Network error: Client connection negotiation failed: client connection to 127.24.128.126:46405: connect: Connection refused (error 111)
16:16:58.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:58.696919 25542 leader_election.cc:334] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 5440fed9b93c45c580cd974996c8187c (127.24.128.125:43165): Network error: Client connection negotiation failed: client connection to 127.24.128.125:43165: connect: Connection refused (error 111)
16:16:58.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:58.697183 25542 leader_election.cc:302] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 48372f8622dc4b11845f2254fcfffc3d; no voters: 1a14e6a5198641059abfde130f5ae6ff, 5440fed9b93c45c580cd974996c8187c
16:16:58.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:58.697799 25635 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
16:16:59.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:59.223726 25610 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:16:59.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:16:59.224056 25610 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:16:59.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:16:59 dist-test-slave-fnz7 krb5kdc[25101](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790439419, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.128.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:16:59.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:16:59.844434 25537 init.cc:295] Successfully reacquired a new kerberos TGT
16:17:00.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:17:00.227088 25610 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:17:00.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:17:00.227464 25610 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:17:00.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:17:00.235864 25523 master_runner.cc:186] Error getting master registration for 127.24.128.126:46405: Network error: Client connection negotiation failed: client connection to 127.24.128.126:46405: connect: Connection refused (error 111),
16:17:00.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:17:00.236503 25523 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:17:00.527 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.128.125:43165(127.24.128.125:43165)] connecting to peer
16:17:00.528 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.128.126:46405(127.24.128.126:46405)] connecting to peer
16:17:00.529 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:17:00.529 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.128.125:43165(127.24.128.125:43165)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:17:00.530 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.128.125:43165: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:17:00.532 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:17:00.532 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.128.126:46405(127.24.128.126:46405)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:17:00.532 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.128.126:46405: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:17:00.535 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927; will retry
16:17:01.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:17:01.230604 25610 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:17:01.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:17:01.230948 25610 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:17:01.487 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.128.125:43165(127.24.128.125:43165)] connecting to peer
16:17:01.488 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.128.126:46405(127.24.128.126:46405)] connecting to peer
16:17:01.489 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:17:01.490 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.128.125:43165(127.24.128.125:43165)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:17:01.490 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.128.125:43165: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:17:01.491 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:17:01.491 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.128.126:46405(127.24.128.126:46405)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:17:01.491 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.128.126:46405: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:17:01.494 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927; will retry
16:17:02.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:17:02.233929 25610 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:17:02.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:17:02.234251 25610 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:17:03.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:17:03.237244 25610 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:17:03.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:17:03.237557 25610 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:17:04.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:17:04.160352 25644 raft_consensus.cc:482] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
16:17:04.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:17:04.160825 25644 raft_consensus.cc:504] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a14e6a5198641059abfde130f5ae6ff" member_type: VOTER last_known_addr { host: "127.24.128.126" port: 46405 } } peers { permanent_uuid: "5440fed9b93c45c580cd974996c8187c" member_type: VOTER last_known_addr { host: "127.24.128.125" port: 43165 } } peers { permanent_uuid: "48372f8622dc4b11845f2254fcfffc3d" member_type: VOTER last_known_addr { host: "127.24.128.124" port: 43927 } }
16:17:04.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:17:04.163170 25644 leader_election.cc:288] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1a14e6a5198641059abfde130f5ae6ff (127.24.128.126:46405), 5440fed9b93c45c580cd974996c8187c (127.24.128.125:43165)
16:17:04.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:17:04.165126 25543 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.24.128.126:46405: connect: Connection refused (error 111) [suppressed 1 similar messages]
16:17:04.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:17:04.168423 25543 leader_election.cc:334] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 1a14e6a5198641059abfde130f5ae6ff (127.24.128.126:46405): Network error: Client connection negotiation failed: client connection to 127.24.128.126:46405: connect: Connection refused (error 111)
16:17:04.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:17:04.168824 25542 leader_election.cc:334] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 5440fed9b93c45c580cd974996c8187c (127.24.128.125:43165): Network error: Client connection negotiation failed: client connection to 127.24.128.125:43165: connect: Connection refused (error 111)
16:17:04.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:17:04.169122 25542 leader_election.cc:302] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 48372f8622dc4b11845f2254fcfffc3d; no voters: 1a14e6a5198641059abfde130f5ae6ff, 5440fed9b93c45c580cd974996c8187c
16:17:04.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:17:04.169704 25644 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
16:17:04.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:17:04.240615 25610 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:17:04.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:17:04.240908 25610 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:17:05.207 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.128.125:43165(127.24.128.125:43165)] connecting to peer
16:17:05.209 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.128.126:46405(127.24.128.126:46405)] connecting to peer
16:17:05.210 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:17:05.211 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.128.125:43165(127.24.128.125:43165)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:17:05.211 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.128.125:43165: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:17:05.212 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:17:05.212 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.128.126:46405(127.24.128.126:46405)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:17:05.213 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.128.126:46405: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:17:05.215 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927; will retry
16:17:05.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:17:05.239548 25523 master_runner.cc:186] Error getting master registration for 127.24.128.126:46405: Network error: Client connection negotiation failed: client connection to 127.24.128.126:46405: connect: Connection refused (error 111),
16:17:05.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:17:05.239914 25523 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:17:05.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:17:05.243721 25610 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:17:05.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:17:05.243986 25610 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:17:06.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:17:06.247255 25610 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:17:06.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:17:06.247589 25610 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:17:07.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:17:07.250509 25610 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:17:07.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:17:07.250778 25610 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:17:07.467 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.128.125:43165(127.24.128.125:43165)] connecting to peer
16:17:07.469 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.128.126:46405(127.24.128.126:46405)] connecting to peer
16:17:07.470 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:17:07.471 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.128.125:43165(127.24.128.125:43165)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:17:07.471 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.128.125:43165: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:17:07.472 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:17:07.472 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.128.126:46405(127.24.128.126:46405)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:17:07.472 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.128.126:46405: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:17:07.475 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927; will retry
16:17:07.647 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.128.125:43165(127.24.128.125:43165)] connecting to peer
16:17:07.649 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.128.126:46405(127.24.128.126:46405)] connecting to peer
16:17:07.650 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:17:07.650 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.128.125:43165(127.24.128.125:43165)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:17:07.650 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.128.125:43165: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:17:07.652 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:17:07.653 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.128.126:46405(127.24.128.126:46405)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:17:07.653 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.128.126:46405: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:17:07.655 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927; will retry
16:17:08.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:17:08.253768 25610 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:17:08.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:17:08.254065 25610 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:17:09.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:17:09.257692 25610 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:17:09.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:17:09.258095 25610 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:17:09.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:17:09 dist-test-slave-fnz7 krb5kdc[25101](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790439429, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.128.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:17:09.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:17:09.846949 25537 init.cc:295] Successfully reacquired a new kerberos TGT
16:17:10.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:17:10.243341 25523 master_runner.cc:186] Error getting master registration for 127.24.128.126:46405: Network error: Client connection negotiation failed: client connection to 127.24.128.126:46405: connect: Connection refused (error 111),
16:17:10.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:17:10.243732 25523 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:17:10.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:17:10.261087 25610 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:17:10.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:17:10.261366 25610 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:17:11.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:17:11.264143 25610 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:17:11.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:17:11.264410 25610 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:17:11.587 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.128.125:43165(127.24.128.125:43165)] connecting to peer
16:17:11.588 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.128.126:46405(127.24.128.126:46405)] connecting to peer
16:17:11.590 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:17:11.590 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.128.125:43165(127.24.128.125:43165)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:17:11.591 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.128.125:43165: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:17:11.592 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:17:11.593 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.128.126:46405(127.24.128.126:46405)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:17:11.593 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.128.126:46405: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:17:11.595 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927; will retry
16:17:12.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:17:12.267426 25610 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:17:12.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:17:12.267750 25610 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:17:12.327 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.128.125:43165(127.24.128.125:43165)] connecting to peer
16:17:12.328 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.128.126:46405(127.24.128.126:46405)] connecting to peer
16:17:12.328 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:17:12.329 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.128.125:43165(127.24.128.125:43165)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:17:12.329 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.128.125:43165: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:17:12.330 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:17:12.330 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.128.126:46405(127.24.128.126:46405)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:17:12.330 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.128.126:46405: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:17:12.333 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927; will retry
16:17:12.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:17:12.531365 25659 raft_consensus.cc:482] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
16:17:12.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:17:12.531818 25659 raft_consensus.cc:504] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a14e6a5198641059abfde130f5ae6ff" member_type: VOTER last_known_addr { host: "127.24.128.126" port: 46405 } } peers { permanent_uuid: "5440fed9b93c45c580cd974996c8187c" member_type: VOTER last_known_addr { host: "127.24.128.125" port: 43165 } } peers { permanent_uuid: "48372f8622dc4b11845f2254fcfffc3d" member_type: VOTER last_known_addr { host: "127.24.128.124" port: 43927 } }
16:17:12.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:17:12.533501 25659 leader_election.cc:288] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1a14e6a5198641059abfde130f5ae6ff (127.24.128.126:46405), 5440fed9b93c45c580cd974996c8187c (127.24.128.125:43165)
16:17:12.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:17:12.535856 25543 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.24.128.126:46405: connect: Connection refused (error 111) [suppressed 1 similar messages]
16:17:12.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:17:12.538713 25542 leader_election.cc:334] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 5440fed9b93c45c580cd974996c8187c (127.24.128.125:43165): Network error: Client connection negotiation failed: client connection to 127.24.128.125:43165: connect: Connection refused (error 111)
16:17:12.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:17:12.539214 25543 leader_election.cc:334] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 1a14e6a5198641059abfde130f5ae6ff (127.24.128.126:46405): Network error: Client connection negotiation failed: client connection to 127.24.128.126:46405: connect: Connection refused (error 111)
16:17:12.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:17:12.539710 25543 leader_election.cc:302] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 48372f8622dc4b11845f2254fcfffc3d; no voters: 1a14e6a5198641059abfde130f5ae6ff, 5440fed9b93c45c580cd974996c8187c
16:17:12.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:17:12.540414 25659 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
16:17:12.828 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.128.125:43165(127.24.128.125:43165)] connecting to peer
16:17:12.829 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.128.126:46405(127.24.128.126:46405)] connecting to peer
16:17:12.829 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:17:12.829 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.128.125:43165(127.24.128.125:43165)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:17:12.829 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.128.125:43165: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:17:12.831 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:17:12.831 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.128.126:46405(127.24.128.126:46405)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:17:12.831 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.128.126:46405: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:17:12.833 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927; will retry
16:17:13.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:17:13.270750 25610 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:17:13.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:17:13.271060 25610 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:17:14.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:17:14.273847 25610 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:17:14.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:17:14.274196 25610 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:17:15.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:17:15.246601 25523 master_runner.cc:186] Error getting master registration for 127.24.128.126:46405: Network error: Client connection negotiation failed: client connection to 127.24.128.126:46405: connect: Connection refused (error 111),
16:17:15.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:17:15.247054 25523 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:17:15.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:17:15.277220 25610 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:17:15.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:17:15.277536 25610 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:17:16.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:17:16.280606 25610 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:17:16.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:17:16.280877 25610 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:17:16.507 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.128.125:43165(127.24.128.125:43165)] connecting to peer
16:17:16.508 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.128.126:46405(127.24.128.126:46405)] connecting to peer
16:17:16.508 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:17:16.508 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.128.125:43165(127.24.128.125:43165)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:17:16.509 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.128.125:43165: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:17:16.509 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:17:16.510 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.128.126:46405(127.24.128.126:46405)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:17:16.510 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.128.126:46405: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:17:16.512 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927; will retry
16:17:17.068 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.128.125:43165(127.24.128.125:43165)] connecting to peer
16:17:17.068 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.128.126:46405(127.24.128.126:46405)] connecting to peer
16:17:17.069 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:17:17.069 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.128.125:43165(127.24.128.125:43165)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:17:17.069 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.128.125:43165: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:17:17.070 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:17:17.070 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.128.126:46405(127.24.128.126:46405)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:17:17.070 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.128.126:46405: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:17:17.072 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927; will retry
16:17:17.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:17:17.283746 25610 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 48372f8622dc4b11845f2254fcfffc3d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:17:17.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:17:17.284039 25610 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:17:18.087 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.128.125:43165(127.24.128.125:43165)] connecting to peer
16:17:18.088 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.128.126:46405(127.24.128.126:46405)] connecting to peer
16:17:18.088 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:17:18.088 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.128.125:43165(127.24.128.125:43165)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:17:18.088 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.128.125:43165: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165
16:17:18.089 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:17:18.090 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.128.126:46405(127.24.128.126:46405)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:17:18.090 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.128.126:46405: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
16:17:18.092 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927; will retry
16:17:18.097 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=28086), Traces: [0ms] refreshing cache from master, [31ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.125:43165, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.126:46405, [102ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.124:43927, [174ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.125:43165: Network error: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165, [185ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.126:46405: Network error: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [605ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.124:43927: OK, [611ms] delaying RPC due to: Service unavailable: Master config (127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [616ms] refreshing cache from master, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.125:43165, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.126:46405, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.124:43927, [623ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.125:43165: Network error: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165, [627ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.126:46405: Network error: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [637ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.124:43927: OK, [638ms] delaying RPC due to: Service unavailable: Master config (127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [656ms] refreshing cache from master, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.125:43165, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.126:46405, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.124:43927, [660ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.125:43165: Network error: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165, [665ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.124:43927: OK, [667ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.126:46405: Network error: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [668ms] delaying RPC due to: Service unavailable: Master config (127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [675ms] refreshing cache from master, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.125:43165, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.126:46405, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.124:43927, [679ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.125:43165: Network error: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165, [682ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.126:46405: Network error: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [684ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.124:43927: OK, [685ms] delaying RPC due to: Service unavailable: Master config (127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [696ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.125:43165, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.126:46405, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.124:43927, [699ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.125:43165: Network error: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165, [704ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.124:43927: OK, [705ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.126:46405: Network error: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [708ms] delaying RPC due to: Service unavailable: Master config (127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [736ms] refreshing cache from master, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.125:43165, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.126:46405, [740ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.124:43927, [741ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.125:43165: Network error: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165, [743ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.126:46405: Network error: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [745ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.124:43927: OK, [746ms] delaying RPC due to: Service unavailable: Master config (127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [775ms] refreshing cache from master, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.125:43165, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.126:46405, [780ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.124:43927, [783ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.125:43165: Network error: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165, [787ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.124:43927: OK, [789ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.126:46405: Network error: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [790ms] delaying RPC due to: Service unavailable: Master config (127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [856ms] refreshing cache from master, [856ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.125:43165, [858ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.126:46405, [860ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.124:43927, [861ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.125:43165: Network error: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165, [863ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.126:46405: Network error: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [865ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.124:43927: OK, [866ms] delaying RPC due to: Service unavailable: Master config (127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [1076ms] refreshing cache from master, [1076ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.125:43165, [1079ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.126:46405, [1079ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.125:43165: Network error: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165, [1081ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.124:43927, [1088ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.126:46405: Network error: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [1090ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.124:43927: OK, [1091ms] delaying RPC due to: Service unavailable: Master config (127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [1176ms] refreshing cache from master, [1176ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.125:43165, [1177ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.126:46405, [1178ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.124:43927, [1180ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.125:43165: Network error: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165, [1183ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.126:46405: Network error: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [1185ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.124:43927: OK, [1186ms] delaying RPC due to: Service unavailable: Master config (127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [1936ms] refreshing cache from master, [1936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.125:43165, [1937ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.126:46405, [1938ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.124:43927, [1939ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.125:43165: Network error: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165, [1940ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.126:46405: Network error: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [1943ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.124:43927: OK, [1943ms] delaying RPC due to: Service unavailable: Master config (127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [2576ms] refreshing cache from master, [2577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.125:43165, [2578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.126:46405, [2579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.124:43927, [2580ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.125:43165: Network error: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165, [2581ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.126:46405: Network error: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [2583ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.124:43927: OK, [2584ms] delaying RPC due to: Service unavailable: Master config (127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [4396ms] refreshing cache from master, [4396ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.125:43165, [4397ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.126:46405, [4398ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.124:43927, [4399ms] too many traces: truncated at 100 traces, deferred=Deferred@499861845(state=PENDING, result=null, callback=(continuation of Deferred@700918764 after retry RPC after error@9286592) -> (continuation of Deferred@1887565461 after retry RPC after error@927434801) -> (continuation of Deferred@659508606 after retry RPC after error@1244785124) -> (continuation of Deferred@905677673 after retry RPC after error@1047960551) -> (continuation of Deferred@1661928938 after retry RPC after error@649379494) -> (continuation of Deferred@1333643738 after retry RPC after error@1163844209) -> (continuation of Deferred@1583720293 after retry RPC after error@1562862465) -> (continuation of Deferred@2083865318 after retry RPC after error@1326158403) -> (continuation of Deferred@1009146440 after retry RPC after error@1518017830) -> (continuation of Deferred@210720509 after retry RPC after error@528827557) -> (continuation of Deferred@301971719 after retry RPC after error@326042675) -> (continuation of Deferred@1696694301 after retry RPC after error@1345601648) -> (continuation of Deferred@1022356720 after retry RPC after error@1472870850) -> (continuation of Deferred@1048738382 after retry RPC after error@1288145454) -> (continuation of Deferred@252471082 after retry RPC after error@151931680) -> (continuation of Deferred@611520581 after retry RPC after error@1419081759) -> (continuation of Deferred@952922256 after retry RPC after error@1138781154) -> (continuation of Deferred@2142087907 after retry RPC after error@483465) -> (continuation of Deferred@1528092905 after retry RPC after error@468391698) -> (continuation of Deferred@159603774 after retry RPC after error@851142319) -> (continuation of Deferred@1335178113 after retry RPC after error@210104038) -> (continuation of Deferred@527704548 after retry RPC after error@1989386939) -> (continuation of Deferred@276598483 after retry RPC after error@867016932) -> (continuation of Deferred@233364596 after retry RPC after error@1711612168), errback=(continuation of Deferred@700918764 after retry RPC after error@9286592) -> (continuation of Deferred@1887565461 after retry RPC after error@927434801) -> (continuation of Deferred@659508606 after retry RPC after error@1244785124) -> (continuation of Deferred@905677673 after retry RPC after error@1047960551) -> (continuation of Deferred@1661928938 after retry RPC after error@649379494) -> (continuation of Deferred@1333643738 after retry RPC after error@1163844209) -> (continuation of Deferred@1583720293 after retry RPC after error@1562862465) -> (continuation of Deferred@2083865318 after retry RPC after error@1326158403) -> (continuation of Deferred@1009146440 after retry RPC after error@1518017830) -> (continuation of Deferred@210720509 after retry RPC after error@528827557) -> (continuation of Deferred@301971719 after retry RPC after error@326042675) -> (continuation of Deferred@1696694301 after retry RPC after error@1345601648) -> (continuation of Deferred@1022356720 after retry RPC after error@1472870850) -> (continuation of Deferred@1048738382 after retry RPC after error@1288145454) -> (continuation of Deferred@252471082 after retry RPC after error@151931680) -> (continuation of Deferred@611520581 after retry RPC after error@1419081759) -> (continuation of Deferred@952922256 after retry RPC after error@1138781154) -> (continuation of Deferred@2142087907 after retry RPC after error@483465) -> (continuation of Deferred@1528092905 after retry RPC after error@468391698) -> (continuation of Deferred@159603774 after retry RPC after error@851142319) -> (continuation of Deferred@1335178113 after retry RPC after error@210104038) -> (continuation of Deferred@527704548 after retry RPC after error@1989386939) -> (continuation of Deferred@276598483 after retry RPC after error@867016932) -> (continuation of Deferred@233364596 after retry RPC after error@1711612168)))
16:17:18.102 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.24.128.125:43165(127.24.128.125:43165)] disconnecting while in state TERMINATED
16:17:18.103 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.24.128.124:43927(127.24.128.124:43927)] disconnecting while in state READY
16:17:18.103 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.24.128.126:46405(127.24.128.126:46405)] disconnecting while in state TERMINATED
16:17:18.109 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
16:17:18.111 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
16:17:18.111 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.24.128.124:43927(127.24.128.124:43927)] handling channelInactive
16:17:18.112 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.128.124:43927(127.24.128.124:43927)] cleaning up while in state READY due to: connection closed
16:17:18.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:17:18.114675 25089 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskkW_qFj/build/tsan/bin/kudu with pid 25523
16:17:19.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-26T16:17:19Z chronyd exiting
16:17:20.337 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
16:17:20.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:17:20.338445 24360 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskkW_qFj/build/tsan/bin/kudu with pid 24639
16:17:20.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:17:20.364133 24360 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskkW_qFj/build/tsan/bin/kudu with pid 24793
16:17:20.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:17:20.392503 24360 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskkW_qFj/build/tsan/bin/kudu with pid 24948
16:17:20.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:17:20.423764 24360 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskkW_qFj/build/tsan/bin/kudu with pid 24372
16:17:20.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:17:20.456085 24360 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskkW_qFj/build/tsan/bin/kudu with pid 24442
16:17:20.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:17:20.488708 24360 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskkW_qFj/build/tsan/bin/kudu with pid 24517
16:17:21.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-26T16:17:21Z chronyd exiting
16:17:22.692 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=28086), Traces: [0ms] refreshing cache from master, [31ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.125:43165, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.126:46405, [102ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.124:43927, [174ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.125:43165: Network error: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165, [185ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.126:46405: Network error: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [605ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.124:43927: OK, [611ms] delaying RPC due to: Service unavailable: Master config (127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [616ms] refreshing cache from master, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.125:43165, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.126:46405, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.124:43927, [623ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.125:43165: Network error: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165, [627ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.126:46405: Network error: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [637ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.124:43927: OK, [638ms] delaying RPC due to: Service unavailable: Master config (127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [656ms] refreshing cache from master, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.125:43165, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.126:46405, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.124:43927, [660ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.125:43165: Network error: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165, [665ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.124:43927: OK, [667ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.126:46405: Network error: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [668ms] delaying RPC due to: Service unavailable: Master config (127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [675ms] refreshing cache from master, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.125:43165, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.126:46405, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.124:43927, [679ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.125:43165: Network error: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165, [682ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.126:46405: Network error: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [684ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.124:43927: OK, [685ms] delaying RPC due to: Service unavailable: Master config (127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [696ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.125:43165, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.126:46405, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.124:43927, [699ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.125:43165: Network error: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165, [704ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.124:43927: OK, [705ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.126:46405: Network error: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [708ms] delaying RPC due to: Service unavailable: Master config (127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [736ms] refreshing cache from master, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.125:43165, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.126:46405, [740ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.124:43927, [741ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.125:43165: Network error: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165, [743ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.126:46405: Network error: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [745ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.124:43927: OK, [746ms] delaying RPC due to: Service unavailable: Master config (127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [775ms] refreshing cache from master, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.125:43165, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.126:46405, [780ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.124:43927, [783ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.125:43165: Network error: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165, [787ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.124:43927: OK, [789ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.126:46405: Network error: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [790ms] delaying RPC due to: Service unavailable: Master config (127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [856ms] refreshing cache from master, [856ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.125:43165, [858ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.126:46405, [860ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.124:43927, [861ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.125:43165: Network error: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165, [863ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.126:46405: Network error: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [865ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.124:43927: OK, [866ms] delaying RPC due to: Service unavailable: Master config (127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [1076ms] refreshing cache from master, [1076ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.125:43165, [1079ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.126:46405, [1079ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.125:43165: Network error: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165, [1081ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.124:43927, [1088ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.126:46405: Network error: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [1090ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.124:43927: OK, [1091ms] delaying RPC due to: Service unavailable: Master config (127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [1176ms] refreshing cache from master, [1176ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.125:43165, [1177ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.126:46405, [1178ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.124:43927, [1180ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.125:43165: Network error: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165, [1183ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.126:46405: Network error: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [1185ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.124:43927: OK, [1186ms] delaying RPC due to: Service unavailable: Master config (127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [1936ms] refreshing cache from master, [1936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.125:43165, [1937ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.126:46405, [1938ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.124:43927, [1939ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.125:43165: Network error: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165, [1940ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.126:46405: Network error: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [1943ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.124:43927: OK, [1943ms] delaying RPC due to: Service unavailable: Master config (127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [2576ms] refreshing cache from master, [2577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.125:43165, [2578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.126:46405, [2579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.124:43927, [2580ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.125:43165: Network error: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165, [2581ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.126:46405: Network error: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [2583ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.124:43927: OK, [2584ms] delaying RPC due to: Service unavailable: Master config (127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [4396ms] refreshing cache from master, [4396ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.125:43165, [4397ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.126:46405, [4398ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.124:43927, [4399ms] too many traces: truncated at 100 traces, deferred=Deferred@499861845(state=PENDING, result=null, callback=(continuation of Deferred@700918764 after retry RPC after error@9286592) -> (continuation of Deferred@1887565461 after retry RPC after error@927434801) -> (continuation of Deferred@659508606 after retry RPC after error@1244785124) -> (continuation of Deferred@905677673 after retry RPC after error@1047960551) -> (continuation of Deferred@1661928938 after retry RPC after error@649379494) -> (continuation of Deferred@1333643738 after retry RPC after error@1163844209) -> (continuation of Deferred@1583720293 after retry RPC after error@1562862465) -> (continuation of Deferred@2083865318 after retry RPC after error@1326158403) -> (continuation of Deferred@1009146440 after retry RPC after error@1518017830) -> (continuation of Deferred@210720509 after retry RPC after error@528827557) -> (continuation of Deferred@301971719 after retry RPC after error@326042675) -> (continuation of Deferred@1696694301 after retry RPC after error@1345601648) -> (continuation of Deferred@1022356720 after retry RPC after error@1472870850) -> (continuation of Deferred@1048738382 after retry RPC after error@1288145454) -> (continuation of Deferred@252471082 after retry RPC after error@151931680) -> (continuation of Deferred@611520581 after retry RPC after error@1419081759) -> (continuation of Deferred@952922256 after retry RPC after error@1138781154) -> (continuation of Deferred@2142087907 after retry RPC after error@483465) -> (continuation of Deferred@1528092905 after retry RPC after error@468391698) -> (continuation of Deferred@159603774 after retry RPC after error@851142319) -> (continuation of Deferred@1335178113 after retry RPC after error@210104038) -> (continuation of Deferred@527704548 after retry RPC after error@1989386939) -> (continuation of Deferred@276598483 after retry RPC after error@867016932) -> (continuation of Deferred@233364596 after retry RPC after error@1711612168), errback=(continuation of Deferred@700918764 after retry RPC after error@9286592) -> (continuation of Deferred@1887565461 after retry RPC after error@927434801) -> (continuation of Deferred@659508606 after retry RPC after error@1244785124) -> (continuation of Deferred@905677673 after retry RPC after error@1047960551) -> (continuation of Deferred@1661928938 after retry RPC after error@649379494) -> (continuation of Deferred@1333643738 after retry RPC after error@1163844209) -> (continuation of Deferred@1583720293 after retry RPC after error@1562862465) -> (continuation of Deferred@2083865318 after retry RPC after error@1326158403) -> (continuation of Deferred@1009146440 after retry RPC after error@1518017830) -> (continuation of Deferred@210720509 after retry RPC after error@528827557) -> (continuation of Deferred@301971719 after retry RPC after error@326042675) -> (continuation of Deferred@1696694301 after retry RPC after error@1345601648) -> (continuation of Deferred@1022356720 after retry RPC after error@1472870850) -> (continuation of Deferred@1048738382 after retry RPC after error@1288145454) -> (continuation of Deferred@252471082 after retry RPC after error@151931680) -> (continuation of Deferred@611520581 after retry RPC after error@1419081759) -> (continuation of Deferred@952922256 after retry RPC after error@1138781154) -> (continuation of Deferred@2142087907 after retry RPC after error@483465) -> (continuation of Deferred@1528092905 after retry RPC after error@468391698) -> (continuation of Deferred@159603774 after retry RPC after error@851142319) -> (continuation of Deferred@1335178113 after retry RPC after error@210104038) -> (continuation of Deferred@527704548 after retry RPC after error@1989386939) -> (continuation of Deferred@276598483 after retry RPC after error@867016932) -> (continuation of Deferred@233364596 after retry RPC after error@1711612168)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.24.128.126:46405
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=28086), Traces: [0ms] refreshing cache from master, [31ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.125:43165, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.126:46405, [102ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.124:43927, [174ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.125:43165: Network error: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165, [185ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.126:46405: Network error: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [605ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.124:43927: OK, [611ms] delaying RPC due to: Service unavailable: Master config (127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [616ms] refreshing cache from master, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.125:43165, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.126:46405, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.124:43927, [623ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.125:43165: Network error: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165, [627ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.126:46405: Network error: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [637ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.124:43927: OK, [638ms] delaying RPC due to: Service unavailable: Master config (127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [656ms] refreshing cache from master, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.125:43165, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.126:46405, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.124:43927, [660ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.125:43165: Network error: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165, [665ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.124:43927: OK, [667ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.126:46405: Network error: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [668ms] delaying RPC due to: Service unavailable: Master config (127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [675ms] refreshing cache from master, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.125:43165, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.126:46405, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.124:43927, [679ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.125:43165: Network error: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165, [682ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.126:46405: Network error: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [684ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.124:43927: OK, [685ms] delaying RPC due to: Service unavailable: Master config (127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [696ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.125:43165, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.126:46405, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.124:43927, [699ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.125:43165: Network error: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165, [704ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.124:43927: OK, [705ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.126:46405: Network error: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [708ms] delaying RPC due to: Service unavailable: Master config (127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [736ms] refreshing cache from master, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.125:43165, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.126:46405, [740ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.124:43927, [741ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.125:43165: Network error: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165, [743ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.126:46405: Network error: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [745ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.124:43927: OK, [746ms] delaying RPC due to: Service unavailable: Master config (127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [775ms] refreshing cache from master, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.125:43165, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.126:46405, [780ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.124:43927, [783ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.125:43165: Network error: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165, [787ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.124:43927: OK, [789ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.126:46405: Network error: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [790ms] delaying RPC due to: Service unavailable: Master config (127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [856ms] refreshing cache from master, [856ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.125:43165, [858ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.126:46405, [860ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.124:43927, [861ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.125:43165: Network error: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165, [863ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.126:46405: Network error: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [865ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.124:43927: OK, [866ms] delaying RPC due to: Service unavailable: Master config (127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [1076ms] refreshing cache from master, [1076ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.125:43165, [1079ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.126:46405, [1079ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.125:43165: Network error: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165, [1081ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.124:43927, [1088ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.126:46405: Network error: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [1090ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.124:43927: OK, [1091ms] delaying RPC due to: Service unavailable: Master config (127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [1176ms] refreshing cache from master, [1176ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.125:43165, [1177ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.126:46405, [1178ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.124:43927, [1180ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.125:43165: Network error: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165, [1183ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.126:46405: Network error: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [1185ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.124:43927: OK, [1186ms] delaying RPC due to: Service unavailable: Master config (127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [1936ms] refreshing cache from master, [1936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.125:43165, [1937ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.126:46405, [1938ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.124:43927, [1939ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.125:43165: Network error: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165, [1940ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.126:46405: Network error: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [1943ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.124:43927: OK, [1943ms] delaying RPC due to: Service unavailable: Master config (127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [2576ms] refreshing cache from master, [2577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.125:43165, [2578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.126:46405, [2579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.124:43927, [2580ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.125:43165: Network error: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165, [2581ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.126:46405: Network error: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [2583ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.124:43927: OK, [2584ms] delaying RPC due to: Service unavailable: Master config (127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [4396ms] refreshing cache from master, [4396ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.125:43165, [4397ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.126:46405, [4398ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.124:43927, [4399ms] too many traces: truncated at 100 traces, deferred=Deferred@499861845(state=PENDING, result=null, callback=(continuation of Deferred@700918764 after retry RPC after error@9286592) -> (continuation of Deferred@1887565461 after retry RPC after error@927434801) -> (continuation of Deferred@659508606 after retry RPC after error@1244785124) -> (continuation of Deferred@905677673 after retry RPC after error@1047960551) -> (continuation of Deferred@1661928938 after retry RPC after error@649379494) -> (continuation of Deferred@1333643738 after retry RPC after error@1163844209) -> (continuation of Deferred@1583720293 after retry RPC after error@1562862465) -> (continuation of Deferred@2083865318 after retry RPC after error@1326158403) -> (continuation of Deferred@1009146440 after retry RPC after error@1518017830) -> (continuation of Deferred@210720509 after retry RPC after error@528827557) -> (continuation of Deferred@301971719 after retry RPC after error@326042675) -> (continuation of Deferred@1696694301 after retry RPC after error@1345601648) -> (continuation of Deferred@1022356720 after retry RPC after error@1472870850) -> (continuation of Deferred@1048738382 after retry RPC after error@1288145454) -> (continuation of Deferred@252471082 after retry RPC after error@151931680) -> (continuation of Deferred@611520581 after retry RPC after error@1419081759) -> (continuation of Deferred@952922256 after retry RPC after error@1138781154) -> (continuation of Deferred@2142087907 after retry RPC after error@483465) -> (continuation of Deferred@1528092905 after retry RPC after error@468391698) -> (continuation of Deferred@159603774 after retry RPC after error@851142319) -> (continuation of Deferred@1335178113 after retry RPC after error@210104038) -> (continuation of Deferred@527704548 after retry RPC after error@1989386939) -> (continuation of Deferred@276598483 after retry RPC after error@867016932) -> (continuation of Deferred@233364596 after retry RPC after error@1711612168), errback=(continuation of Deferred@700918764 after retry RPC after error@9286592) -> (continuation of Deferred@1887565461 after retry RPC after error@927434801) -> (continuation of Deferred@659508606 after retry RPC after error@1244785124) -> (continuation of Deferred@905677673 after retry RPC after error@1047960551) -> (continuation of Deferred@1661928938 after retry RPC after error@649379494) -> (continuation of Deferred@1333643738 after retry RPC after error@1163844209) -> (continuation of Deferred@1583720293 after retry RPC after error@1562862465) -> (continuation of Deferred@2083865318 after retry RPC after error@1326158403) -> (continuation of Deferred@1009146440 after retry RPC after error@1518017830) -> (continuation of Deferred@210720509 after retry RPC after error@528827557) -> (continuation of Deferred@301971719 after retry RPC after error@326042675) -> (continuation of Deferred@1696694301 after retry RPC after error@1345601648) -> (continuation of Deferred@1022356720 after retry RPC after error@1472870850) -> (continuation of Deferred@1048738382 after retry RPC after error@1288145454) -> (continuation of Deferred@252471082 after retry RPC after error@151931680) -> (continuation of Deferred@611520581 after retry RPC after error@1419081759) -> (continuation of Deferred@952922256 after retry RPC after error@1138781154) -> (continuation of Deferred@2142087907 after retry RPC after error@483465) -> (continuation of Deferred@1528092905 after retry RPC after error@468391698) -> (continuation of Deferred@159603774 after retry RPC after error@851142319) -> (continuation of Deferred@1335178113 after retry RPC after error@210104038) -> (continuation of Deferred@527704548 after retry RPC after error@1989386939) -> (continuation of Deferred@276598483 after retry RPC after error@867016932) -> (continuation of Deferred@233364596 after retry RPC after error@1711612168)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.24.128.126:46405
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
16:17:23.379 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
16:17:23.379 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=28086), Traces: [0ms] refreshing cache from master, [31ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.125:43165, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.126:46405, [102ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.124:43927, [174ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.125:43165: Network error: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165, [185ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.126:46405: Network error: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [605ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.124:43927: OK, [611ms] delaying RPC due to: Service unavailable: Master config (127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [616ms] refreshing cache from master, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.125:43165, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.126:46405, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.124:43927, [623ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.125:43165: Network error: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165, [627ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.126:46405: Network error: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [637ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.124:43927: OK, [638ms] delaying RPC due to: Service unavailable: Master config (127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [656ms] refreshing cache from master, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.125:43165, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.126:46405, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.124:43927, [660ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.125:43165: Network error: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165, [665ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.124:43927: OK, [667ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.126:46405: Network error: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [668ms] delaying RPC due to: Service unavailable: Master config (127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [675ms] refreshing cache from master, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.125:43165, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.126:46405, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.124:43927, [679ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.125:43165: Network error: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165, [682ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.126:46405: Network error: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [684ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.124:43927: OK, [685ms] delaying RPC due to: Service unavailable: Master config (127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [696ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.125:43165, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.126:46405, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.124:43927, [699ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.125:43165: Network error: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165, [704ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.124:43927: OK, [705ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.126:46405: Network error: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [708ms] delaying RPC due to: Service unavailable: Master config (127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [736ms] refreshing cache from master, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.125:43165, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.126:46405, [740ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.124:43927, [741ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.125:43165: Network error: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165, [743ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.126:46405: Network error: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [745ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.124:43927: OK, [746ms] delaying RPC due to: Service unavailable: Master config (127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [775ms] refreshing cache from master, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.125:43165, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.126:46405, [780ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.124:43927, [783ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.125:43165: Network error: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165, [787ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.124:43927: OK, [789ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.126:46405: Network error: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [790ms] delaying RPC due to: Service unavailable: Master config (127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [856ms] refreshing cache from master, [856ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.125:43165, [858ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.126:46405, [860ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.124:43927, [861ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.125:43165: Network error: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165, [863ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.126:46405: Network error: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [865ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.124:43927: OK, [866ms] delaying RPC due to: Service unavailable: Master config (127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [1076ms] refreshing cache from master, [1076ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.125:43165, [1079ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.126:46405, [1079ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.125:43165: Network error: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165, [1081ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.124:43927, [1088ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.126:46405: Network error: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [1090ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.124:43927: OK, [1091ms] delaying RPC due to: Service unavailable: Master config (127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [1176ms] refreshing cache from master, [1176ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.125:43165, [1177ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.126:46405, [1178ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.124:43927, [1180ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.125:43165: Network error: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165, [1183ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.126:46405: Network error: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [1185ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.124:43927: OK, [1186ms] delaying RPC due to: Service unavailable: Master config (127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [1936ms] refreshing cache from master, [1936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.125:43165, [1937ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.126:46405, [1938ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.124:43927, [1939ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.125:43165: Network error: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165, [1940ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.126:46405: Network error: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [1943ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.124:43927: OK, [1943ms] delaying RPC due to: Service unavailable: Master config (127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [2576ms] refreshing cache from master, [2577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.125:43165, [2578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.126:46405, [2579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.124:43927, [2580ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.125:43165: Network error: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165, [2581ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.126:46405: Network error: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [2583ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.124:43927: OK, [2584ms] delaying RPC due to: Service unavailable: Master config (127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [4396ms] refreshing cache from master, [4396ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.125:43165, [4397ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.126:46405, [4398ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.124:43927, [4399ms] too many traces: truncated at 100 traces, deferred=Deferred@499861845(state=PENDING, result=null, callback=(continuation of Deferred@700918764 after retry RPC after error@9286592) -> (continuation of Deferred@1887565461 after retry RPC after error@927434801) -> (continuation of Deferred@659508606 after retry RPC after error@1244785124) -> (continuation of Deferred@905677673 after retry RPC after error@1047960551) -> (continuation of Deferred@1661928938 after retry RPC after error@649379494) -> (continuation of Deferred@1333643738 after retry RPC after error@1163844209) -> (continuation of Deferred@1583720293 after retry RPC after error@1562862465) -> (continuation of Deferred@2083865318 after retry RPC after error@1326158403) -> (continuation of Deferred@1009146440 after retry RPC after error@1518017830) -> (continuation of Deferred@210720509 after retry RPC after error@528827557) -> (continuation of Deferred@301971719 after retry RPC after error@326042675) -> (continuation of Deferred@1696694301 after retry RPC after error@1345601648) -> (continuation of Deferred@1022356720 after retry RPC after error@1472870850) -> (continuation of Deferred@1048738382 after retry RPC after error@1288145454) -> (continuation of Deferred@252471082 after retry RPC after error@151931680) -> (continuation of Deferred@611520581 after retry RPC after error@1419081759) -> (continuation of Deferred@952922256 after retry RPC after error@1138781154) -> (continuation of Deferred@2142087907 after retry RPC after error@483465) -> (continuation of Deferred@1528092905 after retry RPC after error@468391698) -> (continuation of Deferred@159603774 after retry RPC after error@851142319) -> (continuation of Deferred@1335178113 after retry RPC after error@210104038) -> (continuation of Deferred@527704548 after retry RPC after error@1989386939) -> (continuation of Deferred@276598483 after retry RPC after error@867016932) -> (continuation of Deferred@233364596 after retry RPC after error@1711612168), errback=(continuation of Deferred@700918764 after retry RPC after error@9286592) -> (continuation of Deferred@1887565461 after retry RPC after error@927434801) -> (continuation of Deferred@659508606 after retry RPC after error@1244785124) -> (continuation of Deferred@905677673 after retry RPC after error@1047960551) -> (continuation of Deferred@1661928938 after retry RPC after error@649379494) -> (continuation of Deferred@1333643738 after retry RPC after error@1163844209) -> (continuation of Deferred@1583720293 after retry RPC after error@1562862465) -> (continuation of Deferred@2083865318 after retry RPC after error@1326158403) -> (continuation of Deferred@1009146440 after retry RPC after error@1518017830) -> (continuation of Deferred@210720509 after retry RPC after error@528827557) -> (continuation of Deferred@301971719 after retry RPC after error@326042675) -> (continuation of Deferred@1696694301 after retry RPC after error@1345601648) -> (continuation of Deferred@1022356720 after retry RPC after error@1472870850) -> (continuation of Deferred@1048738382 after retry RPC after error@1288145454) -> (continuation of Deferred@252471082 after retry RPC after error@151931680) -> (continuation of Deferred@611520581 after retry RPC after error@1419081759) -> (continuation of Deferred@952922256 after retry RPC after error@1138781154) -> (continuation of Deferred@2142087907 after retry RPC after error@483465) -> (continuation of Deferred@1528092905 after retry RPC after error@468391698) -> (continuation of Deferred@159603774 after retry RPC after error@851142319) -> (continuation of Deferred@1335178113 after retry RPC after error@210104038) -> (continuation of Deferred@527704548 after retry RPC after error@1989386939) -> (continuation of Deferred@276598483 after retry RPC after error@867016932) -> (continuation of Deferred@233364596 after retry RPC after error@1711612168)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.24.128.126:46405
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=28086), Traces: [0ms] refreshing cache from master, [31ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.125:43165, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.126:46405, [102ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.124:43927, [174ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.125:43165: Network error: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165, [185ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.126:46405: Network error: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [605ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.124:43927: OK, [611ms] delaying RPC due to: Service unavailable: Master config (127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [616ms] refreshing cache from master, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.125:43165, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.126:46405, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.124:43927, [623ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.125:43165: Network error: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165, [627ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.126:46405: Network error: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [637ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.124:43927: OK, [638ms] delaying RPC due to: Service unavailable: Master config (127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [656ms] refreshing cache from master, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.125:43165, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.126:46405, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.124:43927, [660ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.125:43165: Network error: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165, [665ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.124:43927: OK, [667ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.126:46405: Network error: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [668ms] delaying RPC due to: Service unavailable: Master config (127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [675ms] refreshing cache from master, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.125:43165, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.126:46405, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.124:43927, [679ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.125:43165: Network error: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165, [682ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.126:46405: Network error: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [684ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.124:43927: OK, [685ms] delaying RPC due to: Service unavailable: Master config (127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [696ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.125:43165, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.126:46405, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.124:43927, [699ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.125:43165: Network error: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165, [704ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.124:43927: OK, [705ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.126:46405: Network error: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [708ms] delaying RPC due to: Service unavailable: Master config (127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [736ms] refreshing cache from master, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.125:43165, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.126:46405, [740ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.124:43927, [741ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.125:43165: Network error: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165, [743ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.126:46405: Network error: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [745ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.124:43927: OK, [746ms] delaying RPC due to: Service unavailable: Master config (127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [775ms] refreshing cache from master, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.125:43165, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.126:46405, [780ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.124:43927, [783ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.125:43165: Network error: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165, [787ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.124:43927: OK, [789ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.126:46405: Network error: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [790ms] delaying RPC due to: Service unavailable: Master config (127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [856ms] refreshing cache from master, [856ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.125:43165, [858ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.126:46405, [860ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.124:43927, [861ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.125:43165: Network error: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165, [863ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.126:46405: Network error: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [865ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.124:43927: OK, [866ms] delaying RPC due to: Service unavailable: Master config (127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [1076ms] refreshing cache from master, [1076ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.125:43165, [1079ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.126:46405, [1079ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.125:43165: Network error: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165, [1081ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.124:43927, [1088ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.126:46405: Network error: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [1090ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.124:43927: OK, [1091ms] delaying RPC due to: Service unavailable: Master config (127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [1176ms] refreshing cache from master, [1176ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.125:43165, [1177ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.126:46405, [1178ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.124:43927, [1180ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.125:43165: Network error: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165, [1183ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.126:46405: Network error: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [1185ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.124:43927: OK, [1186ms] delaying RPC due to: Service unavailable: Master config (127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [1936ms] refreshing cache from master, [1936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.125:43165, [1937ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.126:46405, [1938ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.124:43927, [1939ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.125:43165: Network error: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165, [1940ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.126:46405: Network error: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [1943ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.124:43927: OK, [1943ms] delaying RPC due to: Service unavailable: Master config (127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [2576ms] refreshing cache from master, [2577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.125:43165, [2578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.126:46405, [2579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.124:43927, [2580ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.125:43165: Network error: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165, [2581ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.126:46405: Network error: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [2583ms] Sub RPC ConnectToMaster: received response from server master-127.24.128.124:43927: OK, [2584ms] delaying RPC due to: Service unavailable: Master config (127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405, [4396ms] refreshing cache from master, [4396ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.125:43165, [4397ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.126:46405, [4398ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.128.124:43927, [4399ms] too many traces: truncated at 100 traces, deferred=Deferred@499861845(state=PENDING, result=null, callback=(continuation of Deferred@700918764 after retry RPC after error@9286592) -> (continuation of Deferred@1887565461 after retry RPC after error@927434801) -> (continuation of Deferred@659508606 after retry RPC after error@1244785124) -> (continuation of Deferred@905677673 after retry RPC after error@1047960551) -> (continuation of Deferred@1661928938 after retry RPC after error@649379494) -> (continuation of Deferred@1333643738 after retry RPC after error@1163844209) -> (continuation of Deferred@1583720293 after retry RPC after error@1562862465) -> (continuation of Deferred@2083865318 after retry RPC after error@1326158403) -> (continuation of Deferred@1009146440 after retry RPC after error@1518017830) -> (continuation of Deferred@210720509 after retry RPC after error@528827557) -> (continuation of Deferred@301971719 after retry RPC after error@326042675) -> (continuation of Deferred@1696694301 after retry RPC after error@1345601648) -> (continuation of Deferred@1022356720 after retry RPC after error@1472870850) -> (continuation of Deferred@1048738382 after retry RPC after error@1288145454) -> (continuation of Deferred@252471082 after retry RPC after error@151931680) -> (continuation of Deferred@611520581 after retry RPC after error@1419081759) -> (continuation of Deferred@952922256 after retry RPC after error@1138781154) -> (continuation of Deferred@2142087907 after retry RPC after error@483465) -> (continuation of Deferred@1528092905 after retry RPC after error@468391698) -> (continuation of Deferred@159603774 after retry RPC after error@851142319) -> (continuation of Deferred@1335178113 after retry RPC after error@210104038) -> (continuation of Deferred@527704548 after retry RPC after error@1989386939) -> (continuation of Deferred@276598483 after retry RPC after error@867016932) -> (continuation of Deferred@233364596 after retry RPC after error@1711612168), errback=(continuation of Deferred@700918764 after retry RPC after error@9286592) -> (continuation of Deferred@1887565461 after retry RPC after error@927434801) -> (continuation of Deferred@659508606 after retry RPC after error@1244785124) -> (continuation of Deferred@905677673 after retry RPC after error@1047960551) -> (continuation of Deferred@1661928938 after retry RPC after error@649379494) -> (continuation of Deferred@1333643738 after retry RPC after error@1163844209) -> (continuation of Deferred@1583720293 after retry RPC after error@1562862465) -> (continuation of Deferred@2083865318 after retry RPC after error@1326158403) -> (continuation of Deferred@1009146440 after retry RPC after error@1518017830) -> (continuation of Deferred@210720509 after retry RPC after error@528827557) -> (continuation of Deferred@301971719 after retry RPC after error@326042675) -> (continuation of Deferred@1696694301 after retry RPC after error@1345601648) -> (continuation of Deferred@1022356720 after retry RPC after error@1472870850) -> (continuation of Deferred@1048738382 after retry RPC after error@1288145454) -> (continuation of Deferred@252471082 after retry RPC after error@151931680) -> (continuation of Deferred@611520581 after retry RPC after error@1419081759) -> (continuation of Deferred@952922256 after retry RPC after error@1138781154) -> (continuation of Deferred@2142087907 after retry RPC after error@483465) -> (continuation of Deferred@1528092905 after retry RPC after error@468391698) -> (continuation of Deferred@159603774 after retry RPC after error@851142319) -> (continuation of Deferred@1335178113 after retry RPC after error@210104038) -> (continuation of Deferred@527704548 after retry RPC after error@1989386939) -> (continuation of Deferred@276598483 after retry RPC after error@867016932) -> (continuation of Deferred@233364596 after retry RPC after error@1711612168)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.24.128.125:43165,127.24.128.126:46405,127.24.128.124:43927) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.125:43165(127.24.128.125:43165): Connection refused: /127.24.128.125:43165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.128.126:46405(127.24.128.126:46405): Connection refused: /127.24.128.126:46405
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.24.128.126:46405
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)