01:40:09.487 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
01:40:09.496 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task3Ego3M/test-tmp
01:40:09.498 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
01:40:09.502 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task3Ego3M/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
01:40:09.689 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
01:40:09.848 [DEBUG - main] (MiniKuduCluster.java:195) Response:
01:40:09.849 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
01:40:09.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:09.853500 9841 mini_oidc.cc:152] Starting JWKS server
01:40:09.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:09.857771 9841 webserver.cc:460] Webserver started at http://127.0.0.1:39007/ using document root <none> and password file <none>
01:40:09.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:09.858258 9841 mini_oidc.cc:189] Starting OIDC Discovery server
01:40:09.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:09.860152 9841 webserver.cc:460] Webserver started at http://127.0.0.1:33709/ using document root <none> and password file <none>
01:40:09.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-21T01:40:09Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
01:40:09.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-21T01:40:09Z Disabled control of system clock
01:40:09.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:09.906294 9841 external_mini_cluster.cc:1326] Running /tmp/dist-test-task3Ego3M/build/tsan/bin/kudu
01:40:09.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task3Ego3M/build/tsan/bin/kudu
01:40:09.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/master-0/wal
01:40:09.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/master-0/data
01:40:09.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:40:09.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:40:09.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:40:09.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:40:09.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:40:09.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:40:09.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/master-0/logs
01:40:09.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/master-0/data/info.pb
01:40:09.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:40:09.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:40:09.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:40:09.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:40:09.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:40:09.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:40:09.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:40:09.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:40:09.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:40:09.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:40:09.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:40:09.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:40:09.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.156.126:34279
01:40:09.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.156.126
01:40:09.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:40:09.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.156.84:37145
01:40:09.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:40:09.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:40:09.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:40:09.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:40:09.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.156.126:34279,127.9.156.125:33417,127.9.156.124:36323
01:40:09.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:40:09.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39007/jwks.json with env {}
01:40:10.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:10.175604 9853 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:40:10.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:10.176118 9853 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:40:10.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:10.176743 9853 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:40:10.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:10.204430 9853 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:40:10.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:10.204641 9853 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:40:10.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:10.204803 9853 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:40:10.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:10.205004 9853 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:40:10.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:10.205250 9853 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:40:10.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:10.205452 9853 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39007/jwks.json
01:40:10.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:10.235368 9853 master_runner.cc:386] Master server non-default flags:
01:40:10.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:40:10.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.156.84:37145
01:40:10.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:40:10.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:40:10.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/master-0/data
01:40:10.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/master-0/wal
01:40:10.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:40:10.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.156.126:34279,127.9.156.125:33417,127.9.156.124:36323
01:40:10.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:40:10.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:40:10.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:40:10.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.156.126:34279
01:40:10.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:40:10.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:40:10.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:40:10.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39007/jwks.json
01:40:10.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:40:10.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:40:10.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/master-0/data/info.pb
01:40:10.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.156.126
01:40:10.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:40:10.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:40:10.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:40:10.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:40:10.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:40:10.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:40:10.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/master-0/logs
01:40:10.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:40:10.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:40:10.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:40:10.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:40:10.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:40:10.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:40:10.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:40:10.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 00:52:14 UTC on ccycloud.quasar-sdbdje.root.comops.site
01:40:10.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457837
01:40:10.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:40:10.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:10.236668 9853 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:40:10.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:10.238615 9853 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:40:10.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:10.256114 9860 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:40:11.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:10.256151 9859 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:40:11.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:10.256171 9862 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:40:11.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:10.257275 9853 server_base.cc:1034] running on GCE node
01:40:11.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:11.324193 9853 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:40:11.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:11.327090 9853 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:40:11.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:11.328670 9853 hybrid_clock.cc:648] HybridClock initialized: now 1782006011328624 us; error 53 us; skew 500 ppm
01:40:11.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:11.329352 9853 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:40:11.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:11.335721 9853 webserver.cc:460] Webserver started at http://127.9.156.126:41515/ using document root <none> and password file <none>
01:40:11.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:11.336516 9853 fs_manager.cc:359] Metadata directory not provided
01:40:11.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:11.336686 9853 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:40:11.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:11.337039 9853 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:40:11.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:11.341086 9853 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/master-0/data/instance:
01:40:11.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7ead2eb6d0fc47e699ebd29c8dafc7bf"
01:40:11.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:40:11 on dist-test-slave-20xp"
01:40:11.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:11.342015 9853 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/master-0/wal/instance:
01:40:11.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7ead2eb6d0fc47e699ebd29c8dafc7bf"
01:40:11.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:40:11 on dist-test-slave-20xp"
01:40:11.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:11.347993 9853 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.007s sys 0.001s
01:40:11.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:11.352808 9869 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:40:11.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:11.353643 9853 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.001s sys 0.004s
01:40:11.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:11.353930 9853 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/master-0/data,/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/master-0/wal
01:40:11.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7ead2eb6d0fc47e699ebd29c8dafc7bf"
01:40:11.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:40:11 on dist-test-slave-20xp"
01:40:11.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:11.354259 9853 fs_report.cc:383] FS layout report
01:40:11.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:40:11.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/master-0/wal
01:40:11.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/master-0/wal
01:40:11.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/master-0/data/data
01:40:11.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:40:11.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:40:11.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:40:11.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:40:11.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:40:11.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:40:11.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:40:11.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:40:11.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:40:11.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:40:11.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:40:11.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:11.393433 9853 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:40:11.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:11.394636 9853 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:40:11.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:11.395061 9853 kserver.cc:163] Server-wide thread pool size limit: 3276
01:40:11.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:11.460094 9853 rpc_server.cc:307] RPC server started. Bound to: 127.9.156.126:34279
01:40:11.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:11.460170 9920 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.9.156.126:34279 every 8 connection(s)
01:40:11.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:11.462570 9853 server_base.cc:1166] Dumped server information to /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/master-0/data/info.pb
01:40:11.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:11.467603 9921 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
01:40:11.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:11.468063 9841 external_mini_cluster.cc:1388] Started /tmp/dist-test-task3Ego3M/build/tsan/bin/kudu as pid 9853
01:40:11.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:11.468529 9841 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/master-0/wal/instance
01:40:11.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:11.472061 9841 external_mini_cluster.cc:1326] Running /tmp/dist-test-task3Ego3M/build/tsan/bin/kudu
01:40:11.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task3Ego3M/build/tsan/bin/kudu
01:40:11.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/master-1/wal
01:40:11.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/master-1/data
01:40:11.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:40:11.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:40:11.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:40:11.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:40:11.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:40:11.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:40:11.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/master-1/logs
01:40:11.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/master-1/data/info.pb
01:40:11.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:40:11.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:40:11.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:40:11.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:40:11.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:40:11.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:40:11.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:40:11.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:40:11.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:40:11.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:40:11.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:40:11.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:40:11.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.156.125:33417
01:40:11.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.156.125
01:40:11.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:40:11.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.156.84:37145
01:40:11.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:40:11.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:40:11.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:40:11.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:40:11.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.156.126:34279,127.9.156.125:33417,127.9.156.124:36323
01:40:11.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:40:11.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39007/jwks.json with env {}
01:40:11.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:11.480263 9921 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.156.126" port: 34279 } has no permanent_uuid. Determining permanent_uuid...
01:40:11.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:11.507987 9921 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.156.125" port: 33417 } has no permanent_uuid. Determining permanent_uuid...
01:40:11.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:11.509668 9872 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.156.125:33417: connect: Connection refused (error 111)
01:40:11.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:11.513505 9921 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.156.125:33417: Network error: Client connection negotiation failed: client connection to 127.9.156.125:33417: connect: Connection refused (error 111)
01:40:11.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:11.562915 9921 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.156.125" port: 33417 } attempt: 1
01:40:11.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:11.566690 9921 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.156.125:33417: Network error: Client connection negotiation failed: client connection to 127.9.156.125:33417: connect: Connection refused (error 111)
01:40:11.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:11.635140 9921 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.156.125" port: 33417 } attempt: 2
01:40:11.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:11.638983 9921 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.156.125:33417: Network error: Client connection negotiation failed: client connection to 127.9.156.125:33417: connect: Connection refused (error 111)
01:40:11.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:11.730388 9921 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.156.125" port: 33417 } attempt: 3
01:40:11.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:11.733637 9921 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.156.125:33417: Network error: Client connection negotiation failed: client connection to 127.9.156.125:33417: connect: Connection refused (error 111)
01:40:11.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:11.749095 9923 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:40:11.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:11.749681 9923 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:40:11.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:11.750289 9923 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:40:11.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:11.776849 9923 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:40:11.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:11.777204 9923 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:40:11.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:11.777561 9923 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:40:11.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:11.777866 9923 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:40:11.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:11.778157 9923 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:40:11.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:11.778393 9923 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39007/jwks.json
01:40:11.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:11.807646 9923 master_runner.cc:386] Master server non-default flags:
01:40:11.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:40:11.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.156.84:37145
01:40:11.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:40:11.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:40:11.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/master-1/data
01:40:11.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/master-1/wal
01:40:11.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:40:11.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.156.126:34279,127.9.156.125:33417,127.9.156.124:36323
01:40:11.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:40:11.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:40:11.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:40:11.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.156.125:33417
01:40:11.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:40:11.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:40:11.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:40:11.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39007/jwks.json
01:40:11.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:40:11.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:40:11.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/master-1/data/info.pb
01:40:11.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.156.125
01:40:11.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:40:11.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:40:11.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:40:11.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:40:11.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:40:11.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:40:11.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/master-1/logs
01:40:11.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:40:11.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:40:11.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:40:11.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:40:11.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:40:11.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:40:11.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:40:11.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 00:52:14 UTC on ccycloud.quasar-sdbdje.root.comops.site
01:40:11.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457837
01:40:11.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:40:11.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:11.809098 9923 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:40:11.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:11.811105 9923 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:40:11.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:11.826849 9932 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:40:11.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:11.876947 9921 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.156.125" port: 33417 } attempt: 4
01:40:11.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:11.880054 9921 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.156.125:33417: Network error: Client connection negotiation failed: client connection to 127.9.156.125:33417: connect: Connection refused (error 111)
01:40:12.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:12.179414 9921 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.156.125" port: 33417 } attempt: 5
01:40:12.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:12.184556 9921 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.156.125:33417: Network error: Client connection negotiation failed: client connection to 127.9.156.125:33417: connect: Connection refused (error 111)
01:40:12.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:12.732100 9921 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.156.125" port: 33417 } attempt: 6
01:40:12.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:12.744246 9921 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.156.125:33417: Network error: Client connection negotiation failed: client connection to 127.9.156.125:33417: connect: Connection refused (error 111)
01:40:12.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:11.827383 9933 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:40:12.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:11.827008 9935 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:40:12.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:12.941421 9934 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1113 milliseconds
01:40:12.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:12.941524 9923 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
01:40:12.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:12.943405 9923 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:40:12.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:12.947005 9923 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:40:12.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:12.948451 9923 hybrid_clock.cc:648] HybridClock initialized: now 1782006012948385 us; error 118 us; skew 500 ppm
01:40:12.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:12.949226 9923 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:40:12.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:12.954767 9923 webserver.cc:460] Webserver started at http://127.9.156.125:43215/ using document root <none> and password file <none>
01:40:12.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:12.955657 9923 fs_manager.cc:359] Metadata directory not provided
01:40:12.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:12.955864 9923 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:40:12.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:12.956373 9923 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:40:12.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:12.960583 9923 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/master-1/data/instance:
01:40:12.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a0cd55f5cc184908a0ddea432e060024"
01:40:12.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:40:12 on dist-test-slave-20xp"
01:40:12.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:12.961628 9923 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/master-1/wal/instance:
01:40:12.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a0cd55f5cc184908a0ddea432e060024"
01:40:12.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:40:12 on dist-test-slave-20xp"
01:40:12.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:12.967976 9923 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.007s sys 0.001s
01:40:12.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:12.974922 9944 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:40:12.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:12.975888 9923 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
01:40:12.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:12.976189 9923 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/master-1/data,/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/master-1/wal
01:40:12.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a0cd55f5cc184908a0ddea432e060024"
01:40:12.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:40:12 on dist-test-slave-20xp"
01:40:12.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:12.976573 9923 fs_report.cc:383] FS layout report
01:40:12.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:40:12.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/master-1/wal
01:40:12.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/master-1/wal
01:40:12.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/master-1/data/data
01:40:12.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:40:12.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:40:12.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:40:12.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:40:12.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:40:12.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:40:12.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:40:12.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:40:12.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:40:12.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:40:12.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:40:13.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:13.010675 9923 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:40:13.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:13.012012 9923 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:40:13.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:13.012887 9923 kserver.cc:163] Server-wide thread pool size limit: 3276
01:40:13.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:13.080082 9923 rpc_server.cc:307] RPC server started. Bound to: 127.9.156.125:33417
01:40:13.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:13.080168 9995 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.9.156.125:33417 every 8 connection(s)
01:40:13.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:13.082634 9923 server_base.cc:1166] Dumped server information to /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/master-1/data/info.pb
01:40:13.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:13.087129 9841 external_mini_cluster.cc:1388] Started /tmp/dist-test-task3Ego3M/build/tsan/bin/kudu as pid 9923
01:40:13.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:13.087615 9841 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/master-1/wal/instance
01:40:13.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:13.088284 9996 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
01:40:13.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:13.090358 9841 external_mini_cluster.cc:1326] Running /tmp/dist-test-task3Ego3M/build/tsan/bin/kudu
01:40:13.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task3Ego3M/build/tsan/bin/kudu
01:40:13.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/master-2/wal
01:40:13.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/master-2/data
01:40:13.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:40:13.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:40:13.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:40:13.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:40:13.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:40:13.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:40:13.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/master-2/logs
01:40:13.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/master-2/data/info.pb
01:40:13.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:40:13.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:40:13.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:40:13.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:40:13.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:40:13.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:40:13.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:40:13.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:40:13.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:40:13.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:40:13.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:40:13.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:40:13.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.156.124:36323
01:40:13.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.156.124
01:40:13.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:40:13.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.156.84:37145
01:40:13.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:40:13.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:40:13.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:40:13.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:40:13.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.156.126:34279,127.9.156.125:33417,127.9.156.124:36323
01:40:13.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:40:13.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39007/jwks.json with env {}
01:40:13.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:13.103084 9996 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.156.126" port: 34279 } has no permanent_uuid. Determining permanent_uuid...
01:40:13.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:13.128341 9996 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.156.125" port: 33417 } has no permanent_uuid. Determining permanent_uuid...
01:40:13.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:13.142483 9996 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.156.124" port: 36323 } has no permanent_uuid. Determining permanent_uuid...
01:40:13.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:13.144548 9945 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.156.124:36323: connect: Connection refused (error 111)
01:40:13.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:13.148370 9996 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.156.124:36323: Network error: Client connection negotiation failed: client connection to 127.9.156.124:36323: connect: Connection refused (error 111)
01:40:13.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:13.197717 9996 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.156.124" port: 36323 } attempt: 1
01:40:13.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:13.200582 9996 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.156.124:36323: Network error: Client connection negotiation failed: client connection to 127.9.156.124:36323: connect: Connection refused (error 111)
01:40:13.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:13.269018 9996 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.156.124" port: 36323 } attempt: 2
01:40:13.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:13.272845 9996 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.156.124:36323: Network error: Client connection negotiation failed: client connection to 127.9.156.124:36323: connect: Connection refused (error 111)
01:40:13.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:13.364306 9996 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.156.124" port: 36323 } attempt: 3
01:40:13.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:13.368227 9996 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.156.124:36323: Network error: Client connection negotiation failed: client connection to 127.9.156.124:36323: connect: Connection refused (error 111)
01:40:13.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:13.375598 9998 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:40:13.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:13.376005 9998 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:40:13.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:13.376420 9998 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:40:13.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:13.402514 9998 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:40:13.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:13.402735 9998 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:40:13.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:13.402982 9998 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:40:13.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:13.403193 9998 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:40:13.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:13.403494 9998 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:40:13.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:13.403694 9998 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39007/jwks.json
01:40:13.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:13.431530 9998 master_runner.cc:386] Master server non-default flags:
01:40:13.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:40:13.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.156.84:37145
01:40:13.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:40:13.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:40:13.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/master-2/data
01:40:13.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/master-2/wal
01:40:13.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:40:13.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.156.126:34279,127.9.156.125:33417,127.9.156.124:36323
01:40:13.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:40:13.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:40:13.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:40:13.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.156.124:36323
01:40:13.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:40:13.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:40:13.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:40:13.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39007/jwks.json
01:40:13.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:40:13.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:40:13.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/master-2/data/info.pb
01:40:13.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.156.124
01:40:13.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:40:13.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:40:13.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:40:13.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:40:13.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:40:13.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:40:13.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/master-2/logs
01:40:13.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:40:13.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:40:13.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:40:13.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:40:13.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:40:13.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:40:13.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:40:13.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 00:52:14 UTC on ccycloud.quasar-sdbdje.root.comops.site
01:40:13.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457837
01:40:13.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:40:13.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:13.432579 9998 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:40:13.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:13.434059 9998 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:40:13.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:13.451818 10008 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:40:13.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:13.511557 9996 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.156.124" port: 36323 } attempt: 4
01:40:13.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:13.515116 9996 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.156.124:36323: Network error: Client connection negotiation failed: client connection to 127.9.156.124:36323: connect: Connection refused (error 111)
01:40:13.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:13.804719 9921 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.156.125" port: 33417 } attempt: 7
01:40:13.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:13.814630 9996 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.156.124" port: 36323 } attempt: 5
01:40:13.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:13.816623 9921 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.156.124" port: 36323 } has no permanent_uuid. Determining permanent_uuid...
01:40:13.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:13.818642 9996 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.156.124:36323: Network error: Client connection negotiation failed: client connection to 127.9.156.124:36323: connect: Connection refused (error 111)
01:40:13.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:13.821383 9921 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.156.124:36323: Network error: Client connection negotiation failed: client connection to 127.9.156.124:36323: connect: Connection refused (error 111)
01:40:13.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:13.879877 9921 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.156.124" port: 36323 } attempt: 1
01:40:13.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:13.883895 9921 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.156.124:36323: Network error: Client connection negotiation failed: client connection to 127.9.156.124:36323: connect: Connection refused (error 111)
01:40:13.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:13.965260 9921 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.156.124" port: 36323 } attempt: 2
01:40:13.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:13.968843 9921 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.156.124:36323: Network error: Client connection negotiation failed: client connection to 127.9.156.124:36323: connect: Connection refused (error 111)
01:40:14.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:14.054158 9921 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.156.124" port: 36323 } attempt: 3
01:40:14.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:14.057843 9921 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.156.124:36323: Network error: Client connection negotiation failed: client connection to 127.9.156.124:36323: connect: Connection refused (error 111)
01:40:14.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:14.198251 9921 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.156.124" port: 36323 } attempt: 4
01:40:14.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:14.202230 9921 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.156.124:36323: Network error: Client connection negotiation failed: client connection to 127.9.156.124:36323: connect: Connection refused (error 111)
01:40:14.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:14.366616 9996 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.156.124" port: 36323 } attempt: 6
01:40:14.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:14.373394 9996 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.156.124:36323: Network error: Client connection negotiation failed: client connection to 127.9.156.124:36323: connect: Connection refused (error 111)
01:40:14.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:14.485707 9921 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.156.124" port: 36323 } attempt: 5
01:40:14.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:14.489403 9921 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.156.124:36323: Network error: Client connection negotiation failed: client connection to 127.9.156.124:36323: connect: Connection refused (error 111)
01:40:14.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:13.452348 10011 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:40:14.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:13.453234 9998 server_base.cc:1034] running on GCE node
01:40:14.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:13.451840 10009 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:40:14.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:14.523772 9998 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:40:14.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:14.526494 9998 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:40:14.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:14.527802 9998 hybrid_clock.cc:648] HybridClock initialized: now 1782006014527782 us; error 38 us; skew 500 ppm
01:40:14.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:14.528523 9998 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:40:14.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:14.534157 9998 webserver.cc:460] Webserver started at http://127.9.156.124:35417/ using document root <none> and password file <none>
01:40:14.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:14.534868 9998 fs_manager.cc:359] Metadata directory not provided
01:40:14.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:14.535012 9998 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:40:14.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:14.535446 9998 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:40:14.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:14.541447 9998 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/master-2/data/instance:
01:40:14.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bfce06ff6e2845cb9622ecc42e3af8b0"
01:40:14.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:40:14 on dist-test-slave-20xp"
01:40:14.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:14.542963 9998 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/master-2/wal/instance:
01:40:14.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bfce06ff6e2845cb9622ecc42e3af8b0"
01:40:14.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:40:14 on dist-test-slave-20xp"
01:40:14.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:14.551043 9998 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.003s sys 0.004s
01:40:14.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:14.557363 10023 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:40:14.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:14.558456 9998 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.001s
01:40:14.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:14.558756 9998 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/master-2/data,/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/master-2/wal
01:40:14.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bfce06ff6e2845cb9622ecc42e3af8b0"
01:40:14.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:40:14 on dist-test-slave-20xp"
01:40:14.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:14.559098 9998 fs_report.cc:383] FS layout report
01:40:14.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:40:14.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/master-2/wal
01:40:14.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/master-2/wal
01:40:14.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/master-2/data/data
01:40:14.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:40:14.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:40:14.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:40:14.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:40:14.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:40:14.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:40:14.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:40:14.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:40:14.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:40:14.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:40:14.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:40:14.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:14.585824 9998 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:40:14.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:14.586982 9998 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:40:14.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:14.587818 9998 kserver.cc:163] Server-wide thread pool size limit: 3276
01:40:14.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:14.654498 9998 rpc_server.cc:307] RPC server started. Bound to: 127.9.156.124:36323
01:40:14.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:14.654562 10074 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.9.156.124:36323 every 8 connection(s)
01:40:14.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:14.656929 9998 server_base.cc:1166] Dumped server information to /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/master-2/data/info.pb
01:40:14.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:14.662315 10075 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
01:40:14.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:14.664171 9841 external_mini_cluster.cc:1388] Started /tmp/dist-test-task3Ego3M/build/tsan/bin/kudu as pid 9998
01:40:14.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:14.664455 9841 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/master-2/wal/instance
01:40:14.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:14.677943 10075 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.156.126" port: 34279 } has no permanent_uuid. Determining permanent_uuid...
01:40:14.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:14.705730 10075 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.156.125" port: 33417 } has no permanent_uuid. Determining permanent_uuid...
01:40:14.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:14.716964 10075 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.156.124" port: 36323 } has no permanent_uuid. Determining permanent_uuid...
01:40:14.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:14.730537 10075 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P bfce06ff6e2845cb9622ecc42e3af8b0: Bootstrap starting.
01:40:14.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:14.735618 10075 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P bfce06ff6e2845cb9622ecc42e3af8b0: Neither blocks nor log segments found. Creating new log.
01:40:14.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:14.737460 10075 log.cc:824] T 00000000000000000000000000000000 P bfce06ff6e2845cb9622ecc42e3af8b0: Log is configured to *not* fsync() on all Append() calls
01:40:14.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:14.741112 10075 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P bfce06ff6e2845cb9622ecc42e3af8b0: No bootstrap required, opened a new log
01:40:14.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:14.763957 10075 raft_consensus.cc:348] T 00000000000000000000000000000000 P bfce06ff6e2845cb9622ecc42e3af8b0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7ead2eb6d0fc47e699ebd29c8dafc7bf" member_type: VOTER last_known_addr { host: "127.9.156.126" port: 34279 } } peers { permanent_uuid: "a0cd55f5cc184908a0ddea432e060024" member_type: VOTER last_known_addr { host: "127.9.156.125" port: 33417 } } peers { permanent_uuid: "bfce06ff6e2845cb9622ecc42e3af8b0" member_type: VOTER last_known_addr { host: "127.9.156.124" port: 36323 } }
01:40:14.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:14.764473 10075 raft_consensus.cc:374] T 00000000000000000000000000000000 P bfce06ff6e2845cb9622ecc42e3af8b0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
01:40:14.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:14.764739 10075 raft_consensus.cc:729] T 00000000000000000000000000000000 P bfce06ff6e2845cb9622ecc42e3af8b0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: bfce06ff6e2845cb9622ecc42e3af8b0, State: Initialized, Role: FOLLOWER
01:40:14.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:14.765564 10075 consensus_queue.cc:260] T 00000000000000000000000000000000 P bfce06ff6e2845cb9622ecc42e3af8b0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7ead2eb6d0fc47e699ebd29c8dafc7bf" member_type: VOTER last_known_addr { host: "127.9.156.126" port: 34279 } } peers { permanent_uuid: "a0cd55f5cc184908a0ddea432e060024" member_type: VOTER last_known_addr { host: "127.9.156.125" port: 33417 } } peers { permanent_uuid: "bfce06ff6e2845cb9622ecc42e3af8b0" member_type: VOTER last_known_addr { host: "127.9.156.124" port: 36323 } }
01:40:14.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:14.767973 10084 sys_catalog.cc:455] T 00000000000000000000000000000000 P bfce06ff6e2845cb9622ecc42e3af8b0 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7ead2eb6d0fc47e699ebd29c8dafc7bf" member_type: VOTER last_known_addr { host: "127.9.156.126" port: 34279 } } peers { permanent_uuid: "a0cd55f5cc184908a0ddea432e060024" member_type: VOTER last_known_addr { host: "127.9.156.125" port: 33417 } } peers { permanent_uuid: "bfce06ff6e2845cb9622ecc42e3af8b0" member_type: VOTER last_known_addr { host: "127.9.156.124" port: 36323 } } }
01:40:14.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:14.768448 10084 sys_catalog.cc:458] T 00000000000000000000000000000000 P bfce06ff6e2845cb9622ecc42e3af8b0 [sys.catalog]: This master's current role is: FOLLOWER
01:40:14.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:14.769142 10075 sys_catalog.cc:564] T 00000000000000000000000000000000 P bfce06ff6e2845cb9622ecc42e3af8b0 [sys.catalog]: configured and running, proceeding with master startup.
01:40:14.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:14.792389 10095 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bfce06ff6e2845cb9622ecc42e3af8b0: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:40:14.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:14.792698 10095 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:40:14.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:14.788496 9998 master_runner.cc:186] Error getting master registration for 127.9.156.126:34279: OK, instance_id { permanent_uuid: "7ead2eb6d0fc47e699ebd29c8dafc7bf" instance_seqno: 1782006011441820 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
01:40:14.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:14.812891 9998 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:40:14.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:14.841964 10029 tablet.cc:2369] T 00000000000000000000000000000000 P bfce06ff6e2845cb9622ecc42e3af8b0: Can't schedule compaction. Clean time has not been advanced past its initial value.
01:40:14.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:14.904315 10084 raft_consensus.cc:482] T 00000000000000000000000000000000 P bfce06ff6e2845cb9622ecc42e3af8b0 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:40:14.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:14.904736 10084 raft_consensus.cc:504] T 00000000000000000000000000000000 P bfce06ff6e2845cb9622ecc42e3af8b0 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7ead2eb6d0fc47e699ebd29c8dafc7bf" member_type: VOTER last_known_addr { host: "127.9.156.126" port: 34279 } } peers { permanent_uuid: "a0cd55f5cc184908a0ddea432e060024" member_type: VOTER last_known_addr { host: "127.9.156.125" port: 33417 } } peers { permanent_uuid: "bfce06ff6e2845cb9622ecc42e3af8b0" member_type: VOTER last_known_addr { host: "127.9.156.124" port: 36323 } }
01:40:14.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:14.907244 10084 leader_election.cc:288] T 00000000000000000000000000000000 P bfce06ff6e2845cb9622ecc42e3af8b0 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7ead2eb6d0fc47e699ebd29c8dafc7bf (127.9.156.126:34279), a0cd55f5cc184908a0ddea432e060024 (127.9.156.125:33417)
01:40:15.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.041728 9921 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.156.124" port: 36323 } attempt: 6
01:40:15.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.063189 9921 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7ead2eb6d0fc47e699ebd29c8dafc7bf: Bootstrap starting.
01:40:15.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.069396 9921 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 7ead2eb6d0fc47e699ebd29c8dafc7bf: Neither blocks nor log segments found. Creating new log.
01:40:15.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.071444 9921 log.cc:824] T 00000000000000000000000000000000 P 7ead2eb6d0fc47e699ebd29c8dafc7bf: Log is configured to *not* fsync() on all Append() calls
01:40:15.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.076160 9921 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7ead2eb6d0fc47e699ebd29c8dafc7bf: No bootstrap required, opened a new log
01:40:15.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.104311 9921 raft_consensus.cc:348] T 00000000000000000000000000000000 P 7ead2eb6d0fc47e699ebd29c8dafc7bf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7ead2eb6d0fc47e699ebd29c8dafc7bf" member_type: VOTER last_known_addr { host: "127.9.156.126" port: 34279 } } peers { permanent_uuid: "a0cd55f5cc184908a0ddea432e060024" member_type: VOTER last_known_addr { host: "127.9.156.125" port: 33417 } } peers { permanent_uuid: "bfce06ff6e2845cb9622ecc42e3af8b0" member_type: VOTER last_known_addr { host: "127.9.156.124" port: 36323 } }
01:40:15.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.105031 9921 raft_consensus.cc:374] T 00000000000000000000000000000000 P 7ead2eb6d0fc47e699ebd29c8dafc7bf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
01:40:15.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.105255 9921 raft_consensus.cc:729] T 00000000000000000000000000000000 P 7ead2eb6d0fc47e699ebd29c8dafc7bf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7ead2eb6d0fc47e699ebd29c8dafc7bf, State: Initialized, Role: FOLLOWER
01:40:15.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.106040 9921 consensus_queue.cc:260] T 00000000000000000000000000000000 P 7ead2eb6d0fc47e699ebd29c8dafc7bf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7ead2eb6d0fc47e699ebd29c8dafc7bf" member_type: VOTER last_known_addr { host: "127.9.156.126" port: 34279 } } peers { permanent_uuid: "a0cd55f5cc184908a0ddea432e060024" member_type: VOTER last_known_addr { host: "127.9.156.125" port: 33417 } } peers { permanent_uuid: "bfce06ff6e2845cb9622ecc42e3af8b0" member_type: VOTER last_known_addr { host: "127.9.156.124" port: 36323 } }
01:40:15.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.110145 10099 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7ead2eb6d0fc47e699ebd29c8dafc7bf [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7ead2eb6d0fc47e699ebd29c8dafc7bf" member_type: VOTER last_known_addr { host: "127.9.156.126" port: 34279 } } peers { permanent_uuid: "a0cd55f5cc184908a0ddea432e060024" member_type: VOTER last_known_addr { host: "127.9.156.125" port: 33417 } } peers { permanent_uuid: "bfce06ff6e2845cb9622ecc42e3af8b0" member_type: VOTER last_known_addr { host: "127.9.156.124" port: 36323 } } }
01:40:15.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.110816 10099 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7ead2eb6d0fc47e699ebd29c8dafc7bf [sys.catalog]: This master's current role is: FOLLOWER
01:40:15.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.111652 9896 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "bfce06ff6e2845cb9622ecc42e3af8b0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7ead2eb6d0fc47e699ebd29c8dafc7bf" is_pre_election: true
01:40:15.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.117628 9896 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 7ead2eb6d0fc47e699ebd29c8dafc7bf [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate bfce06ff6e2845cb9622ecc42e3af8b0 in term 0.
01:40:15.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.118453 9921 sys_catalog.cc:564] T 00000000000000000000000000000000 P 7ead2eb6d0fc47e699ebd29c8dafc7bf [sys.catalog]: configured and running, proceeding with master startup.
01:40:15.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.118690 10027 leader_election.cc:302] T 00000000000000000000000000000000 P bfce06ff6e2845cb9622ecc42e3af8b0 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7ead2eb6d0fc47e699ebd29c8dafc7bf, bfce06ff6e2845cb9622ecc42e3af8b0; no voters:
01:40:15.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.119484 10084 raft_consensus.cc:2781] T 00000000000000000000000000000000 P bfce06ff6e2845cb9622ecc42e3af8b0 [term 0 FOLLOWER]: Leader pre-election won for term 1
01:40:15.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.119750 10084 raft_consensus.cc:482] T 00000000000000000000000000000000 P bfce06ff6e2845cb9622ecc42e3af8b0 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
01:40:15.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.120021 10084 raft_consensus.cc:3037] T 00000000000000000000000000000000 P bfce06ff6e2845cb9622ecc42e3af8b0 [term 0 FOLLOWER]: Advancing to term 1
01:40:15.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.125581 10084 raft_consensus.cc:504] T 00000000000000000000000000000000 P bfce06ff6e2845cb9622ecc42e3af8b0 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7ead2eb6d0fc47e699ebd29c8dafc7bf" member_type: VOTER last_known_addr { host: "127.9.156.126" port: 34279 } } peers { permanent_uuid: "a0cd55f5cc184908a0ddea432e060024" member_type: VOTER last_known_addr { host: "127.9.156.125" port: 33417 } } peers { permanent_uuid: "bfce06ff6e2845cb9622ecc42e3af8b0" member_type: VOTER last_known_addr { host: "127.9.156.124" port: 36323 } }
01:40:15.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.127103 10084 leader_election.cc:288] T 00000000000000000000000000000000 P bfce06ff6e2845cb9622ecc42e3af8b0 [CANDIDATE]: Term 1 election: Requested vote from peers 7ead2eb6d0fc47e699ebd29c8dafc7bf (127.9.156.126:34279), a0cd55f5cc184908a0ddea432e060024 (127.9.156.125:33417)
01:40:15.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.127776 9896 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "bfce06ff6e2845cb9622ecc42e3af8b0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7ead2eb6d0fc47e699ebd29c8dafc7bf"
01:40:15.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.128213 9896 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 7ead2eb6d0fc47e699ebd29c8dafc7bf [term 0 FOLLOWER]: Advancing to term 1
01:40:15.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.133773 9896 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 7ead2eb6d0fc47e699ebd29c8dafc7bf [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate bfce06ff6e2845cb9622ecc42e3af8b0 in term 1.
01:40:15.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.134745 10027 leader_election.cc:302] T 00000000000000000000000000000000 P bfce06ff6e2845cb9622ecc42e3af8b0 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7ead2eb6d0fc47e699ebd29c8dafc7bf, bfce06ff6e2845cb9622ecc42e3af8b0; no voters:
01:40:15.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.135385 10084 raft_consensus.cc:2781] T 00000000000000000000000000000000 P bfce06ff6e2845cb9622ecc42e3af8b0 [term 1 FOLLOWER]: Leader election won for term 1
01:40:15.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.146198 10084 raft_consensus.cc:686] T 00000000000000000000000000000000 P bfce06ff6e2845cb9622ecc42e3af8b0 [term 1 LEADER]: Becoming Leader. State: Replica: bfce06ff6e2845cb9622ecc42e3af8b0, State: Running, Role: LEADER
01:40:15.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.149461 10084 consensus_queue.cc:237] T 00000000000000000000000000000000 P bfce06ff6e2845cb9622ecc42e3af8b0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7ead2eb6d0fc47e699ebd29c8dafc7bf" member_type: VOTER last_known_addr { host: "127.9.156.126" port: 34279 } } peers { permanent_uuid: "a0cd55f5cc184908a0ddea432e060024" member_type: VOTER last_known_addr { host: "127.9.156.125" port: 33417 } } peers { permanent_uuid: "bfce06ff6e2845cb9622ecc42e3af8b0" member_type: VOTER last_known_addr { host: "127.9.156.124" port: 36323 } }
01:40:15.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:15.152796 9875 tablet.cc:2369] T 00000000000000000000000000000000 P 7ead2eb6d0fc47e699ebd29c8dafc7bf: Can't schedule compaction. Clean time has not been advanced past its initial value.
01:40:15.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.168638 10103 sys_catalog.cc:455] T 00000000000000000000000000000000 P bfce06ff6e2845cb9622ecc42e3af8b0 [sys.catalog]: SysCatalogTable state changed. Reason: New leader bfce06ff6e2845cb9622ecc42e3af8b0. Latest consensus state: current_term: 1 leader_uuid: "bfce06ff6e2845cb9622ecc42e3af8b0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7ead2eb6d0fc47e699ebd29c8dafc7bf" member_type: VOTER last_known_addr { host: "127.9.156.126" port: 34279 } } peers { permanent_uuid: "a0cd55f5cc184908a0ddea432e060024" member_type: VOTER last_known_addr { host: "127.9.156.125" port: 33417 } } peers { permanent_uuid: "bfce06ff6e2845cb9622ecc42e3af8b0" member_type: VOTER last_known_addr { host: "127.9.156.124" port: 36323 } } }
01:40:15.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.169390 10103 sys_catalog.cc:458] T 00000000000000000000000000000000 P bfce06ff6e2845cb9622ecc42e3af8b0 [sys.catalog]: This master's current role is: LEADER
01:40:15.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.175182 10110 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
01:40:15.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.181689 10110 catalog_manager.cc:1484] Initializing Kudu cluster ID...
01:40:15.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.191087 10113 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
01:40:15.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.203755 9896 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 7ead2eb6d0fc47e699ebd29c8dafc7bf [term 1 FOLLOWER]: Refusing update from remote peer bfce06ff6e2845cb9622ecc42e3af8b0: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
01:40:15.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.205127 10103 consensus_queue.cc:1035] T 00000000000000000000000000000000 P bfce06ff6e2845cb9622ecc42e3af8b0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7ead2eb6d0fc47e699ebd29c8dafc7bf" member_type: VOTER last_known_addr { host: "127.9.156.126" port: 34279 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
01:40:15.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.199393 9853 master_runner.cc:186] Error getting master registration for 127.9.156.125:33417: OK, instance_id { permanent_uuid: "a0cd55f5cc184908a0ddea432e060024" instance_seqno: 1782006013061111 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
01:40:15.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.230418 9853 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:40:15.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.239464 10099 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7ead2eb6d0fc47e699ebd29c8dafc7bf [sys.catalog]: SysCatalogTable state changed. Reason: New leader bfce06ff6e2845cb9622ecc42e3af8b0. Latest consensus state: current_term: 1 leader_uuid: "bfce06ff6e2845cb9622ecc42e3af8b0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7ead2eb6d0fc47e699ebd29c8dafc7bf" member_type: VOTER last_known_addr { host: "127.9.156.126" port: 34279 } } peers { permanent_uuid: "a0cd55f5cc184908a0ddea432e060024" member_type: VOTER last_known_addr { host: "127.9.156.125" port: 33417 } } peers { permanent_uuid: "bfce06ff6e2845cb9622ecc42e3af8b0" member_type: VOTER last_known_addr { host: "127.9.156.124" port: 36323 } } }
01:40:15.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.240085 10099 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7ead2eb6d0fc47e699ebd29c8dafc7bf [sys.catalog]: This master's current role is: FOLLOWER
01:40:15.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.245980 10114 mvcc.cc:205] Tried to move back new op lower bound from 7299096638242385920 to 7299096638085070848. Current Snapshot: MvccSnapshot[applied={T|T < 7299096638242385920}]
01:40:15.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.246767 10084 sys_catalog.cc:455] T 00000000000000000000000000000000 P bfce06ff6e2845cb9622ecc42e3af8b0 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "bfce06ff6e2845cb9622ecc42e3af8b0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7ead2eb6d0fc47e699ebd29c8dafc7bf" member_type: VOTER last_known_addr { host: "127.9.156.126" port: 34279 } } peers { permanent_uuid: "a0cd55f5cc184908a0ddea432e060024" member_type: VOTER last_known_addr { host: "127.9.156.125" port: 33417 } } peers { permanent_uuid: "bfce06ff6e2845cb9622ecc42e3af8b0" member_type: VOTER last_known_addr { host: "127.9.156.124" port: 36323 } } }
01:40:15.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.247555 10084 sys_catalog.cc:458] T 00000000000000000000000000000000 P bfce06ff6e2845cb9622ecc42e3af8b0 [sys.catalog]: This master's current role is: LEADER
01:40:15.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.251040 10099 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7ead2eb6d0fc47e699ebd29c8dafc7bf [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "bfce06ff6e2845cb9622ecc42e3af8b0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7ead2eb6d0fc47e699ebd29c8dafc7bf" member_type: VOTER last_known_addr { host: "127.9.156.126" port: 34279 } } peers { permanent_uuid: "a0cd55f5cc184908a0ddea432e060024" member_type: VOTER last_known_addr { host: "127.9.156.125" port: 33417 } } peers { permanent_uuid: "bfce06ff6e2845cb9622ecc42e3af8b0" member_type: VOTER last_known_addr { host: "127.9.156.124" port: 36323 } } }
01:40:15.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.251647 10099 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7ead2eb6d0fc47e699ebd29c8dafc7bf [sys.catalog]: This master's current role is: FOLLOWER
01:40:15.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.253211 10110 catalog_manager.cc:1347] Generated new cluster ID: 1e72ae4efb5448eba1d04b236f5e40d7
01:40:15.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.253437 10110 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
01:40:15.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.286233 10110 catalog_manager.cc:1370] Generated new certificate authority record
01:40:15.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.288597 10110 catalog_manager.cc:1504] Loading token signing keys...
01:40:15.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.315717 10110 catalog_manager.cc:5868] T 00000000000000000000000000000000 P bfce06ff6e2845cb9622ecc42e3af8b0: Generated new TSK 0
01:40:15.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.316658 10110 catalog_manager.cc:1514] Initializing in-progress tserver states...
01:40:15.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.351761 9841 external_mini_cluster.cc:1326] Running /tmp/dist-test-task3Ego3M/build/tsan/bin/kudu
01:40:15.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task3Ego3M/build/tsan/bin/kudu
01:40:15.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/ts-0/wal
01:40:15.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/ts-0/data
01:40:15.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:40:15.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:40:15.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:40:15.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:40:15.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:40:15.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:40:15.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/ts-0/logs
01:40:15.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/ts-0/data/info.pb
01:40:15.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:40:15.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:40:15.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:40:15.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:40:15.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:40:15.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:40:15.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:40:15.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:40:15.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
01:40:15.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:40:15.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.156.65:0
01:40:15.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.156.65
01:40:15.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.156.65
01:40:15.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:40:15.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.156.126:34279,127.9.156.125:33417,127.9.156.124:36323
01:40:15.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.156.84:37145
01:40:15.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:40:15.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:40:15.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
01:40:15.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.433729 9996 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.156.124" port: 36323 } attempt: 7
01:40:15.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.454579 9996 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a0cd55f5cc184908a0ddea432e060024: Bootstrap starting.
01:40:15.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.460309 9996 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P a0cd55f5cc184908a0ddea432e060024: Neither blocks nor log segments found. Creating new log.
01:40:15.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.462421 9996 log.cc:824] T 00000000000000000000000000000000 P a0cd55f5cc184908a0ddea432e060024: Log is configured to *not* fsync() on all Append() calls
01:40:15.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.492784 9996 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a0cd55f5cc184908a0ddea432e060024: No bootstrap required, opened a new log
01:40:15.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.543151 9996 raft_consensus.cc:348] T 00000000000000000000000000000000 P a0cd55f5cc184908a0ddea432e060024 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7ead2eb6d0fc47e699ebd29c8dafc7bf" member_type: VOTER last_known_addr { host: "127.9.156.126" port: 34279 } } peers { permanent_uuid: "a0cd55f5cc184908a0ddea432e060024" member_type: VOTER last_known_addr { host: "127.9.156.125" port: 33417 } } peers { permanent_uuid: "bfce06ff6e2845cb9622ecc42e3af8b0" member_type: VOTER last_known_addr { host: "127.9.156.124" port: 36323 } }
01:40:15.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.543876 9996 raft_consensus.cc:374] T 00000000000000000000000000000000 P a0cd55f5cc184908a0ddea432e060024 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
01:40:15.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.544104 9996 raft_consensus.cc:729] T 00000000000000000000000000000000 P a0cd55f5cc184908a0ddea432e060024 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a0cd55f5cc184908a0ddea432e060024, State: Initialized, Role: FOLLOWER
01:40:15.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.544948 9996 consensus_queue.cc:260] T 00000000000000000000000000000000 P a0cd55f5cc184908a0ddea432e060024 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7ead2eb6d0fc47e699ebd29c8dafc7bf" member_type: VOTER last_known_addr { host: "127.9.156.126" port: 34279 } } peers { permanent_uuid: "a0cd55f5cc184908a0ddea432e060024" member_type: VOTER last_known_addr { host: "127.9.156.125" port: 33417 } } peers { permanent_uuid: "bfce06ff6e2845cb9622ecc42e3af8b0" member_type: VOTER last_known_addr { host: "127.9.156.124" port: 36323 } }
01:40:15.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.549716 9996 sys_catalog.cc:564] T 00000000000000000000000000000000 P a0cd55f5cc184908a0ddea432e060024 [sys.catalog]: configured and running, proceeding with master startup.
01:40:15.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.550189 9969 raft_consensus.cc:3037] T 00000000000000000000000000000000 P a0cd55f5cc184908a0ddea432e060024 [term 0 FOLLOWER]: Advancing to term 1
01:40:15.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.551393 9971 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "bfce06ff6e2845cb9622ecc42e3af8b0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a0cd55f5cc184908a0ddea432e060024" is_pre_election: true
01:40:15.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.553179 9970 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "bfce06ff6e2845cb9622ecc42e3af8b0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a0cd55f5cc184908a0ddea432e060024"
01:40:15.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.561250 9969 raft_consensus.cc:1261] T 00000000000000000000000000000000 P a0cd55f5cc184908a0ddea432e060024 [term 1 FOLLOWER]: Refusing update from remote peer bfce06ff6e2845cb9622ecc42e3af8b0: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
01:40:15.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.548533 10122 sys_catalog.cc:455] T 00000000000000000000000000000000 P a0cd55f5cc184908a0ddea432e060024 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7ead2eb6d0fc47e699ebd29c8dafc7bf" member_type: VOTER last_known_addr { host: "127.9.156.126" port: 34279 } } peers { permanent_uuid: "a0cd55f5cc184908a0ddea432e060024" member_type: VOTER last_known_addr { host: "127.9.156.125" port: 33417 } } peers { permanent_uuid: "bfce06ff6e2845cb9622ecc42e3af8b0" member_type: VOTER last_known_addr { host: "127.9.156.124" port: 36323 } } }
01:40:15.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.562678 10103 consensus_queue.cc:1035] T 00000000000000000000000000000000 P bfce06ff6e2845cb9622ecc42e3af8b0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a0cd55f5cc184908a0ddea432e060024" member_type: VOTER last_known_addr { host: "127.9.156.125" port: 33417 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
01:40:15.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.562634 10122 sys_catalog.cc:458] T 00000000000000000000000000000000 P a0cd55f5cc184908a0ddea432e060024 [sys.catalog]: This master's current role is: FOLLOWER
01:40:15.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.640100 10122 sys_catalog.cc:455] T 00000000000000000000000000000000 P a0cd55f5cc184908a0ddea432e060024 [sys.catalog]: SysCatalogTable state changed. Reason: New leader bfce06ff6e2845cb9622ecc42e3af8b0. Latest consensus state: current_term: 1 leader_uuid: "bfce06ff6e2845cb9622ecc42e3af8b0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7ead2eb6d0fc47e699ebd29c8dafc7bf" member_type: VOTER last_known_addr { host: "127.9.156.126" port: 34279 } } peers { permanent_uuid: "a0cd55f5cc184908a0ddea432e060024" member_type: VOTER last_known_addr { host: "127.9.156.125" port: 33417 } } peers { permanent_uuid: "bfce06ff6e2845cb9622ecc42e3af8b0" member_type: VOTER last_known_addr { host: "127.9.156.124" port: 36323 } } }
01:40:15.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.640753 10122 sys_catalog.cc:458] T 00000000000000000000000000000000 P a0cd55f5cc184908a0ddea432e060024 [sys.catalog]: This master's current role is: FOLLOWER
01:40:15.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.641556 10133 sys_catalog.cc:455] T 00000000000000000000000000000000 P a0cd55f5cc184908a0ddea432e060024 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "bfce06ff6e2845cb9622ecc42e3af8b0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7ead2eb6d0fc47e699ebd29c8dafc7bf" member_type: VOTER last_known_addr { host: "127.9.156.126" port: 34279 } } peers { permanent_uuid: "a0cd55f5cc184908a0ddea432e060024" member_type: VOTER last_known_addr { host: "127.9.156.125" port: 33417 } } peers { permanent_uuid: "bfce06ff6e2845cb9622ecc42e3af8b0" member_type: VOTER last_known_addr { host: "127.9.156.124" port: 36323 } } }
01:40:15.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.642630 10133 sys_catalog.cc:458] T 00000000000000000000000000000000 P a0cd55f5cc184908a0ddea432e060024 [sys.catalog]: This master's current role is: FOLLOWER
01:40:15.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.664765 10138 catalog_manager.cc:1259] Loaded cluster ID: 1e72ae4efb5448eba1d04b236f5e40d7
01:40:15.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.665239 10138 catalog_manager.cc:1552] T 00000000000000000000000000000000 P a0cd55f5cc184908a0ddea432e060024: loading cluster ID for follower catalog manager: success
01:40:15.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.672135 10138 catalog_manager.cc:1574] T 00000000000000000000000000000000 P a0cd55f5cc184908a0ddea432e060024: acquiring CA information for follower catalog manager: success
01:40:15.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.677635 10138 catalog_manager.cc:1602] T 00000000000000000000000000000000 P a0cd55f5cc184908a0ddea432e060024: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
01:40:15.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:15.781296 10119 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:40:15.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:15.781754 10119 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:40:15.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:15.782284 10119 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:40:15.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:15.812379 10119 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:40:15.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:15.813496 10119 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.9.156.65
01:40:15.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.843954 10119 tablet_server_runner.cc:78] Tablet server non-default flags:
01:40:15.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:40:15.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.156.84:37145
01:40:15.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:40:15.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:40:15.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/ts-0/data
01:40:15.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/ts-0/wal
01:40:15.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:40:15.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:40:15.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.156.65:0
01:40:15.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:40:15.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:40:15.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:40:15.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/ts-0/data/info.pb
01:40:15.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.156.65
01:40:15.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:40:15.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.156.126:34279,127.9.156.125:33417,127.9.156.124:36323
01:40:15.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:40:15.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:40:15.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:40:15.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:40:15.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:40:15.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.156.65
01:40:15.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/ts-0/logs
01:40:15.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:40:15.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:40:15.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:40:15.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
01:40:15.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:40:15.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:40:15.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:40:15.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 00:52:14 UTC on ccycloud.quasar-sdbdje.root.comops.site
01:40:15.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457837
01:40:15.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:40:15.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.845475 10119 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:40:15.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.847496 10119 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:40:15.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:15.876377 10145 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:40:16.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:16.142287 10103 sys_catalog.cc:455] T 00000000000000000000000000000000 P bfce06ff6e2845cb9622ecc42e3af8b0 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "bfce06ff6e2845cb9622ecc42e3af8b0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7ead2eb6d0fc47e699ebd29c8dafc7bf" member_type: VOTER last_known_addr { host: "127.9.156.126" port: 34279 } } peers { permanent_uuid: "a0cd55f5cc184908a0ddea432e060024" member_type: VOTER last_known_addr { host: "127.9.156.125" port: 33417 } } peers { permanent_uuid: "bfce06ff6e2845cb9622ecc42e3af8b0" member_type: VOTER last_known_addr { host: "127.9.156.124" port: 36323 } } }
01:40:16.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:16.143247 10103 sys_catalog.cc:458] T 00000000000000000000000000000000 P bfce06ff6e2845cb9622ecc42e3af8b0 [sys.catalog]: This master's current role is: LEADER
01:40:16.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:16.221030 10113 catalog_manager.cc:1259] Loaded cluster ID: 1e72ae4efb5448eba1d04b236f5e40d7
01:40:16.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:16.229934 10113 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 7ead2eb6d0fc47e699ebd29c8dafc7bf: loading cluster ID for follower catalog manager: success
01:40:16.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:16.252686 10113 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 7ead2eb6d0fc47e699ebd29c8dafc7bf: acquiring CA information for follower catalog manager: success
01:40:16.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:16.273393 10113 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 7ead2eb6d0fc47e699ebd29c8dafc7bf: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
01:40:17.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:15.884013 10146 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:40:17.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:15.881542 10148 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:40:17.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:15.884614 10119 server_base.cc:1034] running on GCE node
01:40:17.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:17.291723 10119 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:40:17.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:17.295661 10119 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:40:17.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:17.297282 10119 hybrid_clock.cc:648] HybridClock initialized: now 1782006017297217 us; error 69 us; skew 500 ppm
01:40:17.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:17.298486 10119 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:40:17.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:17.307869 10119 webserver.cc:460] Webserver started at http://127.9.156.65:36049/ using document root <none> and password file <none>
01:40:17.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:17.309074 10119 fs_manager.cc:359] Metadata directory not provided
01:40:17.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:17.309540 10119 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:40:17.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:17.310149 10119 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:40:17.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:17.317083 10119 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/ts-0/data/instance:
01:40:17.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ae1858d7f9204523b0dd5eca94768ae0"
01:40:17.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:40:17 on dist-test-slave-20xp"
01:40:17.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:17.318713 10119 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/ts-0/wal/instance:
01:40:17.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ae1858d7f9204523b0dd5eca94768ae0"
01:40:17.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:40:17 on dist-test-slave-20xp"
01:40:17.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:17.328517 10119 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.006s sys 0.001s
01:40:17.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:17.335119 10158 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:40:17.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:17.336212 10119 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
01:40:17.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:17.336551 10119 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/ts-0/data,/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/ts-0/wal
01:40:17.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ae1858d7f9204523b0dd5eca94768ae0"
01:40:17.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:40:17 on dist-test-slave-20xp"
01:40:17.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:17.336983 10119 fs_report.cc:383] FS layout report
01:40:17.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:40:17.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/ts-0/wal
01:40:17.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/ts-0/wal
01:40:17.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/ts-0/data/data
01:40:17.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:40:17.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:40:17.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:40:17.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:40:17.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:40:17.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:40:17.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:40:17.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:40:17.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:40:17.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:40:17.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:40:17.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:17.393646 10119 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:40:17.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:17.395712 10119 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:40:17.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:17.396260 10119 kserver.cc:163] Server-wide thread pool size limit: 3276
01:40:17.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:17.399761 10119 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
01:40:17.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:17.404675 10119 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
01:40:17.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:17.404945 10119 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
01:40:17.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:17.405253 10119 ts_tablet_manager.cc:610] Registered 0 tablets
01:40:17.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:17.405493 10119 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
01:40:17.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:17.619488 10119 rpc_server.cc:307] RPC server started. Bound to: 127.9.156.65:38959
01:40:17.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:17.619612 10270 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.9.156.65:38959 every 8 connection(s)
01:40:17.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:17.622804 10119 server_base.cc:1166] Dumped server information to /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/ts-0/data/info.pb
01:40:17.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:17.624166 9841 external_mini_cluster.cc:1388] Started /tmp/dist-test-task3Ego3M/build/tsan/bin/kudu as pid 10119
01:40:17.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:17.624630 9841 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/ts-0/wal/instance
01:40:17.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:17.643239 9841 external_mini_cluster.cc:1326] Running /tmp/dist-test-task3Ego3M/build/tsan/bin/kudu
01:40:17.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task3Ego3M/build/tsan/bin/kudu
01:40:17.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/ts-1/wal
01:40:17.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/ts-1/data
01:40:17.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:40:17.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:40:17.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:40:17.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:40:17.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:40:17.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:40:17.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/ts-1/logs
01:40:17.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/ts-1/data/info.pb
01:40:17.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:40:17.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:40:17.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:40:17.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:40:17.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:40:17.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:40:17.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:40:17.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:40:17.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
01:40:17.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:40:17.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.156.66:0
01:40:17.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.156.66
01:40:17.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.156.66
01:40:17.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:40:17.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.156.126:34279,127.9.156.125:33417,127.9.156.124:36323
01:40:17.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.156.84:37145
01:40:17.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:40:17.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:40:17.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
01:40:17.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:17.723101 10271 heartbeater.cc:347] Connected to a master server at 127.9.156.124:36323
01:40:17.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:17.723526 10271 heartbeater.cc:464] Registering TS with master...
01:40:17.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:17.724524 10271 heartbeater.cc:511] Master 127.9.156.124:36323 requested a full tablet report, sending...
01:40:17.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:17.727819 10040 ts_manager.cc:194] Registered new tserver with Master: ae1858d7f9204523b0dd5eca94768ae0 (127.9.156.65:38959)
01:40:17.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:17.730319 10040 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.9.156.65:59839
01:40:17.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:17.753897 10275 heartbeater.cc:347] Connected to a master server at 127.9.156.125:33417
01:40:17.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:17.754181 10275 heartbeater.cc:464] Registering TS with master...
01:40:17.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:17.754837 10275 heartbeater.cc:511] Master 127.9.156.125:33417 requested a full tablet report, sending...
01:40:17.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:17.757351 9960 ts_manager.cc:194] Registered new tserver with Master: ae1858d7f9204523b0dd5eca94768ae0 (127.9.156.65:38959)
01:40:17.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:17.763473 10276 heartbeater.cc:347] Connected to a master server at 127.9.156.126:34279
01:40:17.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:17.763720 10276 heartbeater.cc:464] Registering TS with master...
01:40:17.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:17.764207 10276 heartbeater.cc:511] Master 127.9.156.126:34279 requested a full tablet report, sending...
01:40:17.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:17.767220 9886 ts_manager.cc:194] Registered new tserver with Master: ae1858d7f9204523b0dd5eca94768ae0 (127.9.156.65:38959)
01:40:18.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:18.157195 10272 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:40:18.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:18.157783 10272 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:40:18.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:18.158491 10272 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:40:18.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:18.206542 10272 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:40:18.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:18.207787 10272 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.9.156.66
01:40:18.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:18.242805 10272 tablet_server_runner.cc:78] Tablet server non-default flags:
01:40:18.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:40:18.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.156.84:37145
01:40:18.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:40:18.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:40:18.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/ts-1/data
01:40:18.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/ts-1/wal
01:40:18.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:40:18.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:40:18.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.156.66:0
01:40:18.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:40:18.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:40:18.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:40:18.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/ts-1/data/info.pb
01:40:18.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.156.66
01:40:18.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:40:18.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.156.126:34279,127.9.156.125:33417,127.9.156.124:36323
01:40:18.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:40:18.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:40:18.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:40:18.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:40:18.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:40:18.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.156.66
01:40:18.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/ts-1/logs
01:40:18.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:40:18.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:40:18.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:40:18.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
01:40:18.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:40:18.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:40:18.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:40:18.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 00:52:14 UTC on ccycloud.quasar-sdbdje.root.comops.site
01:40:18.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457837
01:40:18.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:40:18.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:18.243920 10272 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:40:18.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:18.245549 10272 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:40:18.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:18.267130 10288 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:40:18.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:18.734983 10271 heartbeater.cc:503] Master 127.9.156.124:36323 was elected leader, sending a full tablet report...
01:40:19.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:18.267174 10290 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:40:19.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:18.271286 10272 server_base.cc:1034] running on GCE node
01:40:19.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:18.269646 10287 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:40:19.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:19.348215 10272 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:40:19.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:19.350621 10272 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:40:19.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:19.352006 10272 hybrid_clock.cc:648] HybridClock initialized: now 1782006019351968 us; error 39 us; skew 500 ppm
01:40:19.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:19.352648 10272 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:40:19.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:19.358487 10272 webserver.cc:460] Webserver started at http://127.9.156.66:34903/ using document root <none> and password file <none>
01:40:19.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:19.359300 10272 fs_manager.cc:359] Metadata directory not provided
01:40:19.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:19.359465 10272 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:40:19.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:19.359858 10272 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:40:19.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:19.364418 10272 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/ts-1/data/instance:
01:40:19.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8577ebc3b1104bc0853ca34b3244a3a7"
01:40:19.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:40:19 on dist-test-slave-20xp"
01:40:19.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:19.365404 10272 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/ts-1/wal/instance:
01:40:19.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8577ebc3b1104bc0853ca34b3244a3a7"
01:40:19.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:40:19 on dist-test-slave-20xp"
01:40:19.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:19.371937 10272 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.007s sys 0.000s
01:40:19.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:19.376749 10297 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:40:19.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:19.377709 10272 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.005s sys 0.000s
01:40:19.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:19.377979 10272 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/ts-1/data,/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/ts-1/wal
01:40:19.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8577ebc3b1104bc0853ca34b3244a3a7"
01:40:19.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:40:19 on dist-test-slave-20xp"
01:40:19.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:19.378338 10272 fs_report.cc:383] FS layout report
01:40:19.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:40:19.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/ts-1/wal
01:40:19.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/ts-1/wal
01:40:19.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/ts-1/data/data
01:40:19.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:40:19.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:40:19.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:40:19.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:40:19.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:40:19.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:40:19.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:40:19.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:40:19.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:40:19.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:40:19.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:40:19.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:19.409129 10272 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:40:19.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:19.410280 10272 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:40:19.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:19.410605 10272 kserver.cc:163] Server-wide thread pool size limit: 3276
01:40:19.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:19.413009 10272 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
01:40:19.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:19.416209 10272 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
01:40:19.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:19.416369 10272 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
01:40:19.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:19.416581 10272 ts_tablet_manager.cc:610] Registered 0 tablets
01:40:19.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:19.416702 10272 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
01:40:19.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:19.555783 10272 rpc_server.cc:307] RPC server started. Bound to: 127.9.156.66:35181
01:40:19.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:19.555900 10409 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.9.156.66:35181 every 8 connection(s)
01:40:19.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:19.557870 10272 server_base.cc:1166] Dumped server information to /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/ts-1/data/info.pb
01:40:19.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:19.564486 9841 external_mini_cluster.cc:1388] Started /tmp/dist-test-task3Ego3M/build/tsan/bin/kudu as pid 10272
01:40:19.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:19.565033 9841 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/ts-1/wal/instance
01:40:19.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:19.577102 9841 external_mini_cluster.cc:1326] Running /tmp/dist-test-task3Ego3M/build/tsan/bin/kudu
01:40:19.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task3Ego3M/build/tsan/bin/kudu
01:40:19.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/ts-2/wal
01:40:19.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/ts-2/data
01:40:19.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:40:19.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:40:19.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:40:19.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:40:19.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:40:19.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:40:19.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/ts-2/logs
01:40:19.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/ts-2/data/info.pb
01:40:19.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:40:19.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:40:19.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:40:19.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:40:19.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:40:19.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:40:19.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:40:19.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:40:19.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
01:40:19.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:40:19.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.156.67:0
01:40:19.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.156.67
01:40:19.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.156.67
01:40:19.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:40:19.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.156.126:34279,127.9.156.125:33417,127.9.156.124:36323
01:40:19.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.156.84:37145
01:40:19.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:40:19.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:40:19.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
01:40:19.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:19.603767 10411 heartbeater.cc:347] Connected to a master server at 127.9.156.125:33417
01:40:19.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:19.604123 10411 heartbeater.cc:464] Registering TS with master...
01:40:19.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:19.605043 10411 heartbeater.cc:511] Master 127.9.156.125:33417 requested a full tablet report, sending...
01:40:19.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:19.607914 9960 ts_manager.cc:194] Registered new tserver with Master: 8577ebc3b1104bc0853ca34b3244a3a7 (127.9.156.66:35181)
01:40:19.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:19.610576 10412 heartbeater.cc:347] Connected to a master server at 127.9.156.126:34279
01:40:19.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:19.610814 10412 heartbeater.cc:464] Registering TS with master...
01:40:19.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:19.611475 10412 heartbeater.cc:511] Master 127.9.156.126:34279 requested a full tablet report, sending...
01:40:19.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:19.613262 10410 heartbeater.cc:347] Connected to a master server at 127.9.156.124:36323
01:40:19.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:19.613530 10410 heartbeater.cc:464] Registering TS with master...
01:40:19.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:19.613749 9886 ts_manager.cc:194] Registered new tserver with Master: 8577ebc3b1104bc0853ca34b3244a3a7 (127.9.156.66:35181)
01:40:19.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:19.614152 10410 heartbeater.cc:511] Master 127.9.156.124:36323 requested a full tablet report, sending...
01:40:19.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:19.615824 10040 ts_manager.cc:194] Registered new tserver with Master: 8577ebc3b1104bc0853ca34b3244a3a7 (127.9.156.66:35181)
01:40:19.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:19.617569 10040 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.9.156.66:55581
01:40:19.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:19.870520 10420 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:40:19.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:19.870942 10420 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:40:19.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:19.871440 10420 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:40:19.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:19.898707 10420 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:40:19.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:19.899422 10420 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.9.156.67
01:40:19.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:19.926757 10420 tablet_server_runner.cc:78] Tablet server non-default flags:
01:40:19.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:40:19.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.156.84:37145
01:40:19.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:40:19.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:40:19.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/ts-2/data
01:40:19.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/ts-2/wal
01:40:19.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:40:19.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:40:19.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.156.67:0
01:40:19.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:40:19.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:40:19.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:40:19.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/ts-2/data/info.pb
01:40:19.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.156.67
01:40:19.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:40:19.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.156.126:34279,127.9.156.125:33417,127.9.156.124:36323
01:40:19.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:40:19.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:40:19.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:40:19.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:40:19.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:40:19.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.156.67
01:40:19.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/ts-2/logs
01:40:19.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:40:19.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:40:19.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:40:19.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
01:40:19.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:40:19.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:40:19.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:40:19.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 00:52:14 UTC on ccycloud.quasar-sdbdje.root.comops.site
01:40:19.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457837
01:40:19.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:40:19.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:19.927788 10420 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:40:19.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:19.929076 10420 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:40:19.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:19.946817 10428 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:40:20.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:20.621117 10410 heartbeater.cc:503] Master 127.9.156.124:36323 was elected leader, sending a full tablet report...
01:40:21.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:19.947352 10430 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:40:21.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:19.948695 10427 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:40:21.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:21.011139 10429 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
01:40:21.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:21.011266 10420 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
01:40:21.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:21.015270 10420 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:40:21.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:21.017745 10420 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:40:21.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:21.019155 10420 hybrid_clock.cc:648] HybridClock initialized: now 1782006021019112 us; error 62 us; skew 500 ppm
01:40:21.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:21.019912 10420 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:40:21.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:21.030457 10420 webserver.cc:460] Webserver started at http://127.9.156.67:39493/ using document root <none> and password file <none>
01:40:21.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:21.031178 10420 fs_manager.cc:359] Metadata directory not provided
01:40:21.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:21.031404 10420 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:40:21.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:21.031786 10420 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:40:21.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:21.036207 10420 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/ts-2/data/instance:
01:40:21.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a3851419d7bd4e92abdb79c49d56f7ea"
01:40:21.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:40:21 on dist-test-slave-20xp"
01:40:21.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:21.037070 10420 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/ts-2/wal/instance:
01:40:21.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a3851419d7bd4e92abdb79c49d56f7ea"
01:40:21.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:40:21 on dist-test-slave-20xp"
01:40:21.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:21.043136 10420 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.003s sys 0.004s
01:40:21.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:21.047780 10440 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:40:21.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:21.048854 10420 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.002s
01:40:21.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:21.049121 10420 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/ts-2/data,/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/ts-2/wal
01:40:21.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a3851419d7bd4e92abdb79c49d56f7ea"
01:40:21.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:40:21 on dist-test-slave-20xp"
01:40:21.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:21.049373 10420 fs_report.cc:383] FS layout report
01:40:21.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:40:21.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/ts-2/wal
01:40:21.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/ts-2/wal
01:40:21.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/ts-2/data/data
01:40:21.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:40:21.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:40:21.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:40:21.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:40:21.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:40:21.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:40:21.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:40:21.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:40:21.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:40:21.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:40:21.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:40:21.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:21.081665 10420 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:40:21.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:21.082836 10420 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:40:21.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:21.083297 10420 kserver.cc:163] Server-wide thread pool size limit: 3276
01:40:21.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:21.085716 10420 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
01:40:21.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:21.089373 10420 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
01:40:21.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:21.089579 10420 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
01:40:21.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:21.089864 10420 ts_tablet_manager.cc:610] Registered 0 tablets
01:40:21.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:21.090019 10420 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
01:40:21.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:21.224596 10420 rpc_server.cc:307] RPC server started. Bound to: 127.9.156.67:40599
01:40:21.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:21.224681 10553 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.9.156.67:40599 every 8 connection(s)
01:40:21.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:21.226823 10420 server_base.cc:1166] Dumped server information to /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/ts-2/data/info.pb
01:40:21.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:21.228075 9841 external_mini_cluster.cc:1388] Started /tmp/dist-test-task3Ego3M/build/tsan/bin/kudu as pid 10420
01:40:21.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:21.228729 9841 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster4937946844942344380/ts-2/wal/instance
01:40:21.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:21.273108 10555 heartbeater.cc:347] Connected to a master server at 127.9.156.125:33417
01:40:21.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:21.273512 10555 heartbeater.cc:464] Registering TS with master...
01:40:21.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:21.273736 10554 heartbeater.cc:347] Connected to a master server at 127.9.156.124:36323
01:40:21.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:21.274017 10554 heartbeater.cc:464] Registering TS with master...
01:40:21.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:21.274456 10555 heartbeater.cc:511] Master 127.9.156.125:33417 requested a full tablet report, sending...
01:40:21.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:21.275183 10554 heartbeater.cc:511] Master 127.9.156.124:36323 requested a full tablet report, sending...
01:40:21.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:21.275421 10556 heartbeater.cc:347] Connected to a master server at 127.9.156.126:34279
01:40:21.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:21.275692 10556 heartbeater.cc:464] Registering TS with master...
01:40:21.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:21.276340 10556 heartbeater.cc:511] Master 127.9.156.126:34279 requested a full tablet report, sending...
01:40:21.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:21.277294 10040 ts_manager.cc:194] Registered new tserver with Master: a3851419d7bd4e92abdb79c49d56f7ea (127.9.156.67:40599)
01:40:21.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:21.278116 9960 ts_manager.cc:194] Registered new tserver with Master: a3851419d7bd4e92abdb79c49d56f7ea (127.9.156.67:40599)
01:40:21.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:21.278656 9886 ts_manager.cc:194] Registered new tserver with Master: a3851419d7bd4e92abdb79c49d56f7ea (127.9.156.67:40599)
01:40:21.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:21.279193 10040 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.9.156.67:49613
01:40:21.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:21.291695 9841 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
01:40:21.301 [DEBUG - main] (MiniKuduCluster.java:195) Response:
01:40:21.302 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
01:40:21.311 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.9.156.126"
port: 34279
}
bound_http_address {
host: "127.9.156.126"
port: 41515
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.9.156.125"
port: 33417
}
bound_http_address {
host: "127.9.156.125"
port: 43215
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.9.156.124"
port: 36323
}
bound_http_address {
host: "127.9.156.124"
port: 35417
}
}
}
01:40:21.315 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
01:40:21.317 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.9.156.65"
port: 38959
}
bound_http_address {
host: "127.9.156.65"
port: 36049
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.9.156.66"
port: 35181
}
bound_http_address {
host: "127.9.156.66"
port: 34903
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.9.156.67"
port: 40599
}
bound_http_address {
host: "127.9.156.67"
port: 39493
}
}
}
01:40:21.317 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
01:40:21.486 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
01:40:21.592 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task3Ego3M/test-tmp
01:40:21.592 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
01:40:21.593 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task3Ego3M/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
01:40:21.595 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
01:40:21.905 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:40:21.905 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
01:40:21.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:21.908357 10565 mini_oidc.cc:152] Starting JWKS server
01:40:21.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:21.912321 10565 webserver.cc:460] Webserver started at http://127.0.0.1:42747/ using document root <none> and password file <none>
01:40:21.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:21.912928 10565 mini_oidc.cc:189] Starting OIDC Discovery server
01:40:21.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:21.914785 10565 webserver.cc:460] Webserver started at http://127.0.0.1:40387/ using document root <none> and password file <none>
01:40:21.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
01:40:21.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/krb5kdc/principal' for realm 'KRBTEST.COM',
01:40:21.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
01:40:21.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:40:21 dist-test-slave-20xp krb5kdc[10578](info): setting up network...
01:40:21.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
01:40:21.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:40:21 dist-test-slave-20xp krb5kdc[10578](info): set up 2 sockets
01:40:21.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:40:21 dist-test-slave-20xp krb5kdc[10578](info): commencing operation
01:40:21.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
01:40:22.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:22.282507 10554 heartbeater.cc:503] Master 127.9.156.124:36323 was elected leader, sending a full tablet report...
01:40:22.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:22.905045 10565 mini_kdc.cc:121] Time spent starting KDC: real 0.970s user 0.007s sys 0.010s
01:40:22.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
01:40:22.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
01:40:22.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
01:40:22.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
01:40:22.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
01:40:22.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
01:40:22.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
01:40:22.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
01:40:22.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
01:40:22.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
01:40:22.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/krb5kdc/test-user.keytab.
01:40:22.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/krb5kdc/test-user.keytab.
01:40:23.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:40:23 dist-test-slave-20xp krb5kdc[10578](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006023, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:40:23.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
01:40:23.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-21T01:40:23Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
01:40:23.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-21T01:40:23Z Disabled control of system clock
01:40:23.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.10.81.126@KRBTEST.COM; defaulting to no policy
01:40:23.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
01:40:23.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.10.81.126@KRBTEST.COM" created.
01:40:23.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
01:40:23.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.10.81.126 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/krb5kdc/kudu_127.10.81.126.keytab.
01:40:23.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.10.81.126 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/krb5kdc/kudu_127.10.81.126.keytab.
01:40:23.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:23.063469 10565 external_mini_cluster.cc:1326] Running /tmp/dist-test-task3Ego3M/build/tsan/bin/kudu
01:40:23.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task3Ego3M/build/tsan/bin/kudu
01:40:23.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-0/wal
01:40:23.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-0/data
01:40:23.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:40:23.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:40:23.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:40:23.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:40:23.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:40:23.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:40:23.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-0/logs
01:40:23.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-0/data/info.pb
01:40:23.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:40:23.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:40:23.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:40:23.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:40:23.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:40:23.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:40:23.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:40:23.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:40:23.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:40:23.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:40:23.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:40:23.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:40:23.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.81.126:37005
01:40:23.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.81.126
01:40:23.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:40:23.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/krb5kdc/kudu_127.10.81.126.keytab
01:40:23.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.10.81.126
01:40:23.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:40:23.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
01:40:23.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
01:40:23.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.81.84:36591
01:40:23.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:40:23.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:40:23.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:40:23.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:40:23.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975
01:40:23.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:40:23.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42747/jwks.json
01:40:23.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
01:40:23.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:23.345664 10591 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:40:23.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:23.346200 10591 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:40:23.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:23.346788 10591 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:40:23.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:23.376813 10591 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
01:40:23.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:23.377036 10591 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:40:23.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:23.377384 10591 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:40:23.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:23.377630 10591 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:40:23.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:23.377879 10591 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:40:23.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:23.378067 10591 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:40:23.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:23.378202 10591 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42747/jwks.json
01:40:23.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:23.408648 10591 master_runner.cc:386] Master server non-default flags:
01:40:23.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:40:23.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.81.84:36591
01:40:23.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:40:23.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:40:23.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-0/data
01:40:23.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-0/wal
01:40:23.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
01:40:23.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:40:23.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975
01:40:23.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/krb5kdc/kudu_127.10.81.126.keytab
01:40:23.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.10.81.126
01:40:23.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:40:23.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:40:23.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:40:23.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.81.126:37005
01:40:23.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:40:23.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:40:23.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:40:23.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42747/jwks.json
01:40:23.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:40:23.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
01:40:23.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
01:40:23.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:40:23.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:40:23.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-0/data/info.pb
01:40:23.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.81.126
01:40:23.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:40:23.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:40:23.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:40:23.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:40:23.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:40:23.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:40:23.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-0/logs
01:40:23.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:40:23.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:40:23.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:40:23.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:40:23.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:40:23.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:40:23.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:40:23.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 00:52:14 UTC on ccycloud.quasar-sdbdje.root.comops.site
01:40:23.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457837
01:40:23.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:40:23.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:23.409584 10591 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:40:23.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:23.410370 10591 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:40:23.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:23.426493 10601 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:40:24.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:23.426542 10599 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:40:24.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:23.428680 10591 server_base.cc:1034] running on GCE node
01:40:24.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:23.426553 10598 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:40:24.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:24.504290 10591 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:40:24.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:24.506887 10591 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:40:24.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:24.508256 10591 hybrid_clock.cc:648] HybridClock initialized: now 1782006024508209 us; error 58 us; skew 500 ppm
01:40:24.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:24.508978 10591 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:40:24.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:40:24 dist-test-slave-20xp krb5kdc[10578](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006024, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.81.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:40:24.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:24.514300 10591 init.cc:377] Logged in from keytab as kudu/127.10.81.126@KRBTEST.COM (short username kudu)
01:40:24.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:24.519186 10591 webserver.cc:460] Webserver started at http://127.10.81.126:40327/ using document root <none> and password file <none>
01:40:24.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:24.520092 10591 fs_manager.cc:359] Metadata directory not provided
01:40:24.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:24.520287 10591 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:40:24.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:24.520750 10591 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:40:24.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:24.524830 10591 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-0/data/instance:
01:40:24.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "02bdac1791354adfb4823bed9e4e2e56"
01:40:24.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:40:24 on dist-test-slave-20xp"
01:40:24.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:24.525784 10591 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-0/wal/instance:
01:40:24.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "02bdac1791354adfb4823bed9e4e2e56"
01:40:24.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:40:24 on dist-test-slave-20xp"
01:40:24.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:24.531996 10591 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.003s sys 0.005s
01:40:24.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:24.536850 10609 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:40:24.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:24.537698 10591 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.005s sys 0.000s
01:40:24.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:24.538038 10591 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-0/data,/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-0/wal
01:40:24.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "02bdac1791354adfb4823bed9e4e2e56"
01:40:24.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:40:24 on dist-test-slave-20xp"
01:40:24.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:24.538359 10591 fs_report.cc:383] FS layout report
01:40:24.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:40:24.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-0/wal
01:40:24.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-0/wal
01:40:24.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-0/data/data
01:40:24.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:40:24.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:40:24.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:40:24.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:40:24.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:40:24.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:40:24.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:40:24.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:40:24.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:40:24.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:40:24.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:40:24.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:24.576508 10591 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:40:24.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:24.581911 10591 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:40:24.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:24.582373 10591 kserver.cc:163] Server-wide thread pool size limit: 3276
01:40:24.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:24.652526 10591 rpc_server.cc:307] RPC server started. Bound to: 127.10.81.126:37005
01:40:24.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:24.652547 10660 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.81.126:37005 every 8 connection(s)
01:40:24.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:24.655632 10591 server_base.cc:1166] Dumped server information to /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-0/data/info.pb
01:40:24.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:24.660842 10565 external_mini_cluster.cc:1388] Started /tmp/dist-test-task3Ego3M/build/tsan/bin/kudu as pid 10591
01:40:24.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:24.661295 10565 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-0/wal/instance
01:40:24.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:24.662009 10661 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
01:40:24.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:24.676139 10661 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.81.126" port: 37005 } has no permanent_uuid. Determining permanent_uuid...
01:40:24.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.10.81.125@KRBTEST.COM; defaulting to no policy
01:40:24.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
01:40:24.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.10.81.125@KRBTEST.COM" created.
01:40:24.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:40:24 dist-test-slave-20xp krb5kdc[10578](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006024, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.81.126@KRBTEST.COM for kudu/127.10.81.126@KRBTEST.COM
01:40:24.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
01:40:24.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.10.81.125 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/krb5kdc/kudu_127.10.81.125.keytab.
01:40:24.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.10.81.125 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/krb5kdc/kudu_127.10.81.125.keytab.
01:40:24.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:24.707947 10565 external_mini_cluster.cc:1326] Running /tmp/dist-test-task3Ego3M/build/tsan/bin/kudu
01:40:24.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task3Ego3M/build/tsan/bin/kudu
01:40:24.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-1/wal
01:40:24.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-1/data
01:40:24.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:40:24.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:40:24.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:40:24.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:40:24.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:40:24.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:40:24.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-1/logs
01:40:24.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-1/data/info.pb
01:40:24.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:40:24.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:40:24.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:40:24.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:40:24.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:40:24.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:40:24.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:40:24.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:40:24.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:40:24.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:40:24.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:40:24.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:40:24.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.81.125:38443
01:40:24.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.81.125
01:40:24.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:40:24.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/krb5kdc/kudu_127.10.81.125.keytab
01:40:24.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.10.81.125
01:40:24.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:40:24.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
01:40:24.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
01:40:24.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.81.84:36591
01:40:24.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:40:24.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:40:24.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:40:24.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:40:24.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975
01:40:24.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:40:24.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42747/jwks.json
01:40:24.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
01:40:24.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:24.713409 10661 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.81.125" port: 38443 } has no permanent_uuid. Determining permanent_uuid...
01:40:24.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:24.715524 10611 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.81.125:38443: connect: Connection refused (error 111)
01:40:24.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:24.719358 10661 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.81.125:38443: Network error: Client connection negotiation failed: client connection to 127.10.81.125:38443: connect: Connection refused (error 111)
01:40:24.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:24.768814 10661 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.81.125" port: 38443 } attempt: 1
01:40:24.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:24.772321 10661 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.81.125:38443: Network error: Client connection negotiation failed: client connection to 127.10.81.125:38443: connect: Connection refused (error 111)
01:40:24.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:24.840695 10661 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.81.125" port: 38443 } attempt: 2
01:40:24.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:24.844120 10661 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.81.125:38443: Network error: Client connection negotiation failed: client connection to 127.10.81.125:38443: connect: Connection refused (error 111)
01:40:24.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:24.935503 10661 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.81.125" port: 38443 } attempt: 3
01:40:24.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:24.938665 10661 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.81.125:38443: Network error: Client connection negotiation failed: client connection to 127.10.81.125:38443: connect: Connection refused (error 111)
01:40:24.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:24.980733 10667 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:40:24.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:24.981137 10667 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:40:24.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:24.981590 10667 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:40:25.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:25.008278 10667 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
01:40:25.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:25.008466 10667 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:40:25.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:25.008725 10667 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:40:25.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:25.008898 10667 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:40:25.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:25.009040 10667 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:40:25.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:25.009182 10667 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:40:25.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:25.009299 10667 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42747/jwks.json
01:40:25.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:25.039507 10667 master_runner.cc:386] Master server non-default flags:
01:40:25.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:40:25.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.81.84:36591
01:40:25.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:40:25.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:40:25.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-1/data
01:40:25.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-1/wal
01:40:25.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
01:40:25.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:40:25.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975
01:40:25.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/krb5kdc/kudu_127.10.81.125.keytab
01:40:25.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.10.81.125
01:40:25.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:40:25.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:40:25.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:40:25.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.81.125:38443
01:40:25.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:40:25.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:40:25.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:40:25.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42747/jwks.json
01:40:25.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:40:25.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
01:40:25.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
01:40:25.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:40:25.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:40:25.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-1/data/info.pb
01:40:25.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.81.125
01:40:25.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:40:25.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:40:25.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:40:25.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:40:25.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:40:25.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:40:25.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-1/logs
01:40:25.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:40:25.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:40:25.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:40:25.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:40:25.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:40:25.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:40:25.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:40:25.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 00:52:14 UTC on ccycloud.quasar-sdbdje.root.comops.site
01:40:25.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457837
01:40:25.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:40:25.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:25.040437 10667 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:40:25.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:25.041210 10667 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:40:25.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:25.056661 10674 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:40:25.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:25.081997 10661 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.81.125" port: 38443 } attempt: 4
01:40:25.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:25.087478 10661 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.81.125:38443: Network error: Client connection negotiation failed: client connection to 127.10.81.125:38443: connect: Connection refused (error 111)
01:40:25.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:25.386941 10661 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.81.125" port: 38443 } attempt: 5
01:40:25.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:25.390141 10661 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.81.125:38443: Network error: Client connection negotiation failed: client connection to 127.10.81.125:38443: connect: Connection refused (error 111)
01:40:25.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:25.937646 10661 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.81.125" port: 38443 } attempt: 6
01:40:25.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:25.943816 10661 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.81.125:38443: Network error: Client connection negotiation failed: client connection to 127.10.81.125:38443: connect: Connection refused (error 111)
01:40:26.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:25.056691 10675 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:40:26.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:25.057888 10677 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:40:26.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:26.149725 10676 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
01:40:26.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:26.149859 10667 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
01:40:26.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:26.153669 10667 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:40:26.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:26.155769 10667 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:40:26.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:26.157058 10667 hybrid_clock.cc:648] HybridClock initialized: now 1782006026157027 us; error 51 us; skew 500 ppm
01:40:26.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:26.157737 10667 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:40:26.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:40:26 dist-test-slave-20xp krb5kdc[10578](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006026, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.81.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:40:26.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:26.166564 10667 init.cc:377] Logged in from keytab as kudu/127.10.81.125@KRBTEST.COM (short username kudu)
01:40:26.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:26.171373 10667 webserver.cc:460] Webserver started at http://127.10.81.125:37035/ using document root <none> and password file <none>
01:40:26.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:26.172405 10667 fs_manager.cc:359] Metadata directory not provided
01:40:26.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:26.172655 10667 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:40:26.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:26.173133 10667 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:40:26.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:26.178129 10667 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-1/data/instance:
01:40:26.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "29fc73334b354730b94f4afda636e1a5"
01:40:26.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:40:26 on dist-test-slave-20xp"
01:40:26.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:26.179157 10667 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-1/wal/instance:
01:40:26.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "29fc73334b354730b94f4afda636e1a5"
01:40:26.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:40:26 on dist-test-slave-20xp"
01:40:26.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:26.185259 10667 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.005s sys 0.000s
01:40:26.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:26.189898 10688 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:40:26.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:26.190670 10667 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
01:40:26.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:26.190951 10667 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-1/data,/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-1/wal
01:40:26.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "29fc73334b354730b94f4afda636e1a5"
01:40:26.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:40:26 on dist-test-slave-20xp"
01:40:26.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:26.191265 10667 fs_report.cc:383] FS layout report
01:40:26.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:40:26.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-1/wal
01:40:26.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-1/wal
01:40:26.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-1/data/data
01:40:26.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:40:26.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:40:26.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:40:26.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:40:26.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:40:26.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:40:26.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:40:26.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:40:26.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:40:26.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:40:26.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:40:26.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:26.230885 10667 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:40:26.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:26.236325 10667 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:40:26.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:26.236698 10667 kserver.cc:163] Server-wide thread pool size limit: 3276
01:40:26.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:26.307168 10667 rpc_server.cc:307] RPC server started. Bound to: 127.10.81.125:38443
01:40:26.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:26.307242 10739 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.81.125:38443 every 8 connection(s)
01:40:26.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:26.309602 10667 server_base.cc:1166] Dumped server information to /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-1/data/info.pb
01:40:26.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:26.315045 10740 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
01:40:26.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:26.317687 10565 external_mini_cluster.cc:1388] Started /tmp/dist-test-task3Ego3M/build/tsan/bin/kudu as pid 10667
01:40:26.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:26.318033 10565 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-1/wal/instance
01:40:26.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:26.325212 10740 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.81.126" port: 37005 } has no permanent_uuid. Determining permanent_uuid...
01:40:26.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.10.81.124@KRBTEST.COM; defaulting to no policy
01:40:26.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
01:40:26.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.10.81.124@KRBTEST.COM" created.
01:40:26.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:40:26 dist-test-slave-20xp krb5kdc[10578](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006026, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.81.125@KRBTEST.COM for kudu/127.10.81.126@KRBTEST.COM
01:40:26.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
01:40:26.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.10.81.124 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/krb5kdc/kudu_127.10.81.124.keytab.
01:40:26.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.10.81.124 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/krb5kdc/kudu_127.10.81.124.keytab.
01:40:26.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:26.357631 10565 external_mini_cluster.cc:1326] Running /tmp/dist-test-task3Ego3M/build/tsan/bin/kudu
01:40:26.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task3Ego3M/build/tsan/bin/kudu
01:40:26.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-2/wal
01:40:26.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-2/data
01:40:26.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:40:26.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:40:26.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:40:26.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:40:26.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:40:26.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:40:26.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-2/logs
01:40:26.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-2/data/info.pb
01:40:26.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:40:26.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:40:26.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:40:26.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:40:26.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:40:26.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:40:26.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:40:26.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:40:26.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:40:26.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:40:26.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:40:26.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:40:26.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.81.124:43975
01:40:26.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.81.124
01:40:26.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:40:26.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/krb5kdc/kudu_127.10.81.124.keytab
01:40:26.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.10.81.124
01:40:26.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:40:26.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
01:40:26.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
01:40:26.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.81.84:36591
01:40:26.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:40:26.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:40:26.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:40:26.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:40:26.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975
01:40:26.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:40:26.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42747/jwks.json
01:40:26.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
01:40:26.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:26.359447 10740 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.81.125" port: 38443 } has no permanent_uuid. Determining permanent_uuid...
01:40:26.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:40:26 dist-test-slave-20xp krb5kdc[10578](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006026, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.81.125@KRBTEST.COM for kudu/127.10.81.125@KRBTEST.COM
01:40:26.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:26.383633 10740 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.81.124" port: 43975 } has no permanent_uuid. Determining permanent_uuid...
01:40:26.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:26.385505 10692 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.81.124:43975: connect: Connection refused (error 111)
01:40:26.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:26.388721 10740 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.81.124:43975: Network error: Client connection negotiation failed: client connection to 127.10.81.124:43975: connect: Connection refused (error 111)
01:40:26.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:26.438110 10740 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.81.124" port: 43975 } attempt: 1
01:40:26.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:26.441634 10740 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.81.124:43975: Network error: Client connection negotiation failed: client connection to 127.10.81.124:43975: connect: Connection refused (error 111)
01:40:26.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:26.509958 10740 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.81.124" port: 43975 } attempt: 2
01:40:26.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:26.513003 10740 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.81.124:43975: Network error: Client connection negotiation failed: client connection to 127.10.81.124:43975: connect: Connection refused (error 111)
01:40:26.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:26.604321 10740 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.81.124" port: 43975 } attempt: 3
01:40:26.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:26.607757 10740 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.81.124:43975: Network error: Client connection negotiation failed: client connection to 127.10.81.124:43975: connect: Connection refused (error 111)
01:40:26.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:26.643739 10747 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:40:26.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:26.644182 10747 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:40:26.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:26.644649 10747 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:40:26.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:26.685683 10747 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
01:40:26.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:26.685878 10747 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:40:26.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:26.686094 10747 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:40:26.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:26.686367 10747 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:40:26.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:26.686651 10747 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:40:26.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:26.686960 10747 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:40:26.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:26.687139 10747 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42747/jwks.json
01:40:26.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:26.719775 10747 master_runner.cc:386] Master server non-default flags:
01:40:26.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:40:26.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.81.84:36591
01:40:26.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:40:26.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:40:26.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-2/data
01:40:26.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-2/wal
01:40:26.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
01:40:26.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:40:26.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975
01:40:26.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/krb5kdc/kudu_127.10.81.124.keytab
01:40:26.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.10.81.124
01:40:26.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:40:26.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:40:26.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:40:26.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.81.124:43975
01:40:26.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:40:26.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:40:26.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:40:26.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42747/jwks.json
01:40:26.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:40:26.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
01:40:26.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
01:40:26.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:40:26.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:40:26.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-2/data/info.pb
01:40:26.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.81.124
01:40:26.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:40:26.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:40:26.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:40:26.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:40:26.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:40:26.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:40:26.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-2/logs
01:40:26.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:40:26.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:40:26.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:40:26.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:40:26.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:40:26.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:40:26.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:40:26.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 00:52:14 UTC on ccycloud.quasar-sdbdje.root.comops.site
01:40:26.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457837
01:40:26.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:40:26.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:26.720847 10747 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:40:26.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:26.721700 10747 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:40:26.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:26.738598 10754 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:40:26.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:26.751133 10740 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.81.124" port: 43975 } attempt: 4
01:40:26.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:26.754232 10740 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.81.124:43975: Network error: Client connection negotiation failed: client connection to 127.10.81.124:43975: connect: Connection refused (error 111)
01:40:27.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:27.004166 10661 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.81.125" port: 38443 } attempt: 7
01:40:27.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:40:27 dist-test-slave-20xp krb5kdc[10578](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006024, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.81.126@KRBTEST.COM for kudu/127.10.81.125@KRBTEST.COM
01:40:27.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:27.033821 10661 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.81.124" port: 43975 } has no permanent_uuid. Determining permanent_uuid...
01:40:27.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:27.039523 10661 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.81.124:43975: Network error: Client connection negotiation failed: client connection to 127.10.81.124:43975: connect: Connection refused (error 111)
01:40:27.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:27.053723 10740 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.81.124" port: 43975 } attempt: 5
01:40:27.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:27.056926 10740 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.81.124:43975: Network error: Client connection negotiation failed: client connection to 127.10.81.124:43975: connect: Connection refused (error 111)
01:40:27.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:27.097879 10661 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.81.124" port: 43975 } attempt: 1
01:40:27.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:27.101553 10661 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.81.124:43975: Network error: Client connection negotiation failed: client connection to 127.10.81.124:43975: connect: Connection refused (error 111)
01:40:27.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:27.182928 10661 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.81.124" port: 43975 } attempt: 2
01:40:27.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:27.186748 10661 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.81.124:43975: Network error: Client connection negotiation failed: client connection to 127.10.81.124:43975: connect: Connection refused (error 111)
01:40:27.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:27.272099 10661 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.81.124" port: 43975 } attempt: 3
01:40:27.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:27.275847 10661 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.81.124:43975: Network error: Client connection negotiation failed: client connection to 127.10.81.124:43975: connect: Connection refused (error 111)
01:40:27.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:27.416294 10661 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.81.124" port: 43975 } attempt: 4
01:40:27.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:27.420166 10661 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.81.124:43975: Network error: Client connection negotiation failed: client connection to 127.10.81.124:43975: connect: Connection refused (error 111)
01:40:27.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:27.604413 10740 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.81.124" port: 43975 } attempt: 6
01:40:27.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:27.610702 10740 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.81.124:43975: Network error: Client connection negotiation failed: client connection to 127.10.81.124:43975: connect: Connection refused (error 111)
01:40:27.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:27.703497 10661 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.81.124" port: 43975 } attempt: 5
01:40:27.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:27.706964 10661 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.81.124:43975: Network error: Client connection negotiation failed: client connection to 127.10.81.124:43975: connect: Connection refused (error 111)
01:40:27.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:26.739060 10755 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:40:27.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:26.739362 10757 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:40:27.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:27.847628 10756 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
01:40:27.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:27.847746 10747 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
01:40:27.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:27.851326 10747 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:40:27.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:27.853993 10747 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:40:27.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:27.855315 10747 hybrid_clock.cc:648] HybridClock initialized: now 1782006027855254 us; error 70 us; skew 500 ppm
01:40:27.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:27.856231 10747 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:40:27.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:40:27 dist-test-slave-20xp krb5kdc[10578](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006027, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.81.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:40:27.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:27.861896 10747 init.cc:377] Logged in from keytab as kudu/127.10.81.124@KRBTEST.COM (short username kudu)
01:40:27.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:27.866196 10747 webserver.cc:460] Webserver started at http://127.10.81.124:41835/ using document root <none> and password file <none>
01:40:27.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:27.866930 10747 fs_manager.cc:359] Metadata directory not provided
01:40:27.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:27.867084 10747 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:40:27.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:27.867470 10747 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:40:27.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:27.871910 10747 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-2/data/instance:
01:40:27.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7c78c4f5682e4bfd96481a43439d7e92"
01:40:27.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:40:27 on dist-test-slave-20xp"
01:40:27.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:27.872948 10747 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-2/wal/instance:
01:40:27.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7c78c4f5682e4bfd96481a43439d7e92"
01:40:27.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:40:27 on dist-test-slave-20xp"
01:40:27.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:27.879583 10747 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.003s sys 0.005s
01:40:27.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:27.884323 10770 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:40:27.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:27.885339 10747 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.003s
01:40:27.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:27.885627 10747 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-2/data,/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-2/wal
01:40:27.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7c78c4f5682e4bfd96481a43439d7e92"
01:40:27.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:40:27 on dist-test-slave-20xp"
01:40:27.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:27.885906 10747 fs_report.cc:383] FS layout report
01:40:27.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:40:27.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-2/wal
01:40:27.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-2/wal
01:40:27.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-2/data/data
01:40:27.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:40:27.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:40:27.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:40:27.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:40:27.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:40:27.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:40:27.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:40:27.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:40:27.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:40:27.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:40:27.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:40:27.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:27.918330 10747 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:40:27.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:27.923321 10747 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:40:27.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:27.923674 10747 kserver.cc:163] Server-wide thread pool size limit: 3276
01:40:27.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:27.992594 10747 rpc_server.cc:307] RPC server started. Bound to: 127.10.81.124:43975
01:40:27.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:27.992693 10821 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.81.124:43975 every 8 connection(s)
01:40:27.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:27.995097 10747 server_base.cc:1166] Dumped server information to /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-2/data/info.pb
01:40:28.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:28.000594 10565 external_mini_cluster.cc:1388] Started /tmp/dist-test-task3Ego3M/build/tsan/bin/kudu as pid 10747
01:40:28.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:28.001035 10565 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-2/wal/instance
01:40:28.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:28.001516 10822 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
01:40:28.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:28.021797 10822 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.81.126" port: 37005 } has no permanent_uuid. Determining permanent_uuid...
01:40:28.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:40:28 dist-test-slave-20xp krb5kdc[10578](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006023, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.10.81.124@KRBTEST.COM
01:40:28.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:40:28 dist-test-slave-20xp krb5kdc[10578](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006027, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.81.124@KRBTEST.COM for kudu/127.10.81.126@KRBTEST.COM
01:40:28.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:28.050624 10822 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.81.125" port: 38443 } has no permanent_uuid. Determining permanent_uuid...
01:40:28.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:40:28 dist-test-slave-20xp krb5kdc[10578](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006023, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.10.81.125@KRBTEST.COM
01:40:28.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:40:28 dist-test-slave-20xp krb5kdc[10578](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006027, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.81.124@KRBTEST.COM for kudu/127.10.81.125@KRBTEST.COM
01:40:28.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:28.075145 10822 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.81.124" port: 43975 } has no permanent_uuid. Determining permanent_uuid...
01:40:28.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:40:28 dist-test-slave-20xp krb5kdc[10578](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006023, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.10.81.126@KRBTEST.COM
01:40:28.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:40:28 dist-test-slave-20xp krb5kdc[10578](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006027, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.81.124@KRBTEST.COM for kudu/127.10.81.124@KRBTEST.COM
01:40:28.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:28.107515 10822 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92: Bootstrap starting.
01:40:28.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:28.107940 10565 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
01:40:28.114 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:40:28.116 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
01:40:28.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:28.114567 10822 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92: Neither blocks nor log segments found. Creating new log.
01:40:28.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:28.116407 10822 log.cc:824] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92: Log is configured to *not* fsync() on all Append() calls
01:40:28.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:28.122002 10822 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92: No bootstrap required, opened a new log
01:40:28.127 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
01:40:28.128 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
01:40:28.131 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.10.81.126"
port: 37005
}
bound_http_address {
host: "127.10.81.126"
port: 40327
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.10.81.125"
port: 38443
}
bound_http_address {
host: "127.10.81.125"
port: 37035
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.10.81.124"
port: 43975
}
bound_http_address {
host: "127.10.81.124"
port: 41835
}
}
}
01:40:28.131 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
01:40:28.131 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:40:28.131 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
01:40:28.132 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
01:40:28.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:28.140565 10822 raft_consensus.cc:348] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02bdac1791354adfb4823bed9e4e2e56" member_type: VOTER last_known_addr { host: "127.10.81.126" port: 37005 } } peers { permanent_uuid: "29fc73334b354730b94f4afda636e1a5" member_type: VOTER last_known_addr { host: "127.10.81.125" port: 38443 } } peers { permanent_uuid: "7c78c4f5682e4bfd96481a43439d7e92" member_type: VOTER last_known_addr { host: "127.10.81.124" port: 43975 } }
01:40:28.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:28.141032 10822 raft_consensus.cc:374] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
01:40:28.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:28.141196 10822 raft_consensus.cc:729] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7c78c4f5682e4bfd96481a43439d7e92, State: Initialized, Role: FOLLOWER
01:40:28.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:28.141705 10822 consensus_queue.cc:260] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02bdac1791354adfb4823bed9e4e2e56" member_type: VOTER last_known_addr { host: "127.10.81.126" port: 37005 } } peers { permanent_uuid: "29fc73334b354730b94f4afda636e1a5" member_type: VOTER last_known_addr { host: "127.10.81.125" port: 38443 } } peers { permanent_uuid: "7c78c4f5682e4bfd96481a43439d7e92" member_type: VOTER last_known_addr { host: "127.10.81.124" port: 43975 } }
01:40:28.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:28.144115 10833 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02bdac1791354adfb4823bed9e4e2e56" member_type: VOTER last_known_addr { host: "127.10.81.126" port: 37005 } } peers { permanent_uuid: "29fc73334b354730b94f4afda636e1a5" member_type: VOTER last_known_addr { host: "127.10.81.125" port: 38443 } } peers { permanent_uuid: "7c78c4f5682e4bfd96481a43439d7e92" member_type: VOTER last_known_addr { host: "127.10.81.124" port: 43975 } } }
01:40:28.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:28.144873 10833 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92 [sys.catalog]: This master's current role is: FOLLOWER
01:40:28.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:28.145586 10822 sys_catalog.cc:564] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92 [sys.catalog]: configured and running, proceeding with master startup.
01:40:28.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:40:28 dist-test-slave-20xp krb5kdc[10578](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006028, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:40:28.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
01:40:28.165 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:40:28.167 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/krb5kdc/krb5cc
01:40:28.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:28.168761 10845 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:40:28.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:28.169180 10845 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:40:28.174 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
01:40:28.175 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/krb5kdc/krb5cc
01:40:28.176 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
01:40:28.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:28.178138 10776 tablet.cc:2369] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92: Can't schedule compaction. Clean time has not been advanced past its initial value.
01:40:28.179 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
01:40:28.180 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
01:40:28.180 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
01:40:28.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:28.165253 10747 master_runner.cc:186] Error getting master registration for 127.10.81.126:37005: OK, instance_id { permanent_uuid: "02bdac1791354adfb4823bed9e4e2e56" instance_seqno: 1782006024634059 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
01:40:28.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:28.196209 10747 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:40:28.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:40:28 dist-test-slave-20xp krb5kdc[10578](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006028, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:40:28.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
01:40:28.206 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:40:28.206 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/krb5kdc/krb5cc
01:40:28.208 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
01:40:28.208 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.10.81.126:37005
01:40:28.210 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
01:40:28.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:28.210597 10565 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task3Ego3M/build/tsan/bin/kudu with pid 10591
01:40:28.238 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:40:28.238 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.10.81.125:38443
01:40:28.238 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
01:40:28.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:28.238804 10565 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task3Ego3M/build/tsan/bin/kudu with pid 10667
01:40:28.261 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:40:28.261 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.10.81.124:43975
01:40:28.261 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
01:40:28.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:28.262123 10565 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task3Ego3M/build/tsan/bin/kudu with pid 10747
01:40:28.282 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:40:28.283 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.10.81.126:37005
01:40:28.284 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
01:40:28.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:28.286338 10565 external_mini_cluster.cc:1326] Running /tmp/dist-test-task3Ego3M/build/tsan/bin/kudu
01:40:28.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task3Ego3M/build/tsan/bin/kudu
01:40:28.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-0/wal
01:40:28.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-0/data
01:40:28.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:40:28.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:40:28.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:40:28.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:40:28.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:40:28.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:40:28.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-0/logs
01:40:28.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-0/data/info.pb
01:40:28.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:40:28.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:40:28.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:40:28.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:40:28.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:40:28.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:40:28.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:40:28.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:40:28.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:40:28.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:40:28.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:40:28.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:40:28.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.81.126:37005
01:40:28.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.81.126
01:40:28.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40327
01:40:28.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/krb5kdc/kudu_127.10.81.126.keytab
01:40:28.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.10.81.126
01:40:28.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:40:28.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
01:40:28.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
01:40:28.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.81.84:36591
01:40:28.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:40:28.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:40:28.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:40:28.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:40:28.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975
01:40:28.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:40:28.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42747/jwks.json
01:40:28.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
01:40:28.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:28.559911 10849 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:40:28.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:28.560510 10849 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:40:28.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:28.561157 10849 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:40:28.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:28.592221 10849 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
01:40:28.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:28.592420 10849 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:40:28.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:28.592736 10849 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:40:28.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:28.592935 10849 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:40:28.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:28.593096 10849 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:40:28.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:28.593253 10849 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:40:28.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:28.593379 10849 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42747/jwks.json
01:40:28.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:28.626910 10849 master_runner.cc:386] Master server non-default flags:
01:40:28.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:40:28.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.81.84:36591
01:40:28.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:40:28.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:40:28.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-0/data
01:40:28.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-0/wal
01:40:28.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
01:40:28.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:40:28.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975
01:40:28.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/krb5kdc/kudu_127.10.81.126.keytab
01:40:28.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.10.81.126
01:40:28.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:40:28.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:40:28.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:40:28.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.81.126:37005
01:40:28.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:40:28.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:40:28.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:40:28.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42747/jwks.json
01:40:28.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:40:28.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
01:40:28.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
01:40:28.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:40:28.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:40:28.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-0/data/info.pb
01:40:28.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.81.126
01:40:28.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40327
01:40:28.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:40:28.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:40:28.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:40:28.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:40:28.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:40:28.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-0/logs
01:40:28.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:40:28.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:40:28.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:40:28.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:40:28.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:40:28.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:40:28.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:40:28.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 00:52:14 UTC on ccycloud.quasar-sdbdje.root.comops.site
01:40:28.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457837
01:40:28.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:40:28.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:28.627971 10849 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:40:28.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:28.628901 10849 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:40:28.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:28.644771 10858 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:40:29.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:28.644784 10856 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:40:29.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:28.644824 10855 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:40:29.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:29.723647 10857 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
01:40:29.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:29.723965 10849 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
01:40:29.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:29.728242 10849 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:40:29.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:29.731256 10849 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:40:29.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:29.732681 10849 hybrid_clock.cc:648] HybridClock initialized: now 1782006029732646 us; error 49 us; skew 500 ppm
01:40:29.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:29.733464 10849 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:40:29.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:40:29 dist-test-slave-20xp krb5kdc[10578](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006029, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.81.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:40:29.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:29.738312 10849 init.cc:377] Logged in from keytab as kudu/127.10.81.126@KRBTEST.COM (short username kudu)
01:40:29.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:29.742924 10849 webserver.cc:460] Webserver started at http://127.10.81.126:40327/ using document root <none> and password file <none>
01:40:29.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:29.743719 10849 fs_manager.cc:359] Metadata directory not provided
01:40:29.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:29.743932 10849 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:40:29.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:29.750818 10849 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.006s sys 0.001s
01:40:29.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:29.754405 10866 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:40:29.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:29.755323 10849 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
01:40:29.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:29.755549 10849 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-0/data,/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-0/wal
01:40:29.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "02bdac1791354adfb4823bed9e4e2e56"
01:40:29.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:40:24 on dist-test-slave-20xp"
01:40:29.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:29.757397 10849 fs_report.cc:383] FS layout report
01:40:29.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:40:29.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-0/wal
01:40:29.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-0/wal
01:40:29.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-0/data/data
01:40:29.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:40:29.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:40:29.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:40:29.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:40:29.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:40:29.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:40:29.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:40:29.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:40:29.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:40:29.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:40:29.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:40:29.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:29.790956 10849 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:40:29.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:29.796234 10849 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:40:29.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:29.796586 10849 kserver.cc:163] Server-wide thread pool size limit: 3276
01:40:29.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:29.862187 10849 rpc_server.cc:307] RPC server started. Bound to: 127.10.81.126:37005
01:40:29.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:29.862289 10919 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.81.126:37005 every 8 connection(s)
01:40:29.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:29.864758 10849 server_base.cc:1166] Dumped server information to /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-0/data/info.pb
01:40:29.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:29.874485 10565 external_mini_cluster.cc:1388] Started /tmp/dist-test-task3Ego3M/build/tsan/bin/kudu as pid 10849
01:40:29.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:29.874979 10920 sys_catalog.cc:263] Verifying existing consensus state
01:40:29.875 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:40:29.875 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.10.81.125:38443
01:40:29.875 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
01:40:29.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260621 01:40:29.875900 10920 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
01:40:29.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:29.876354 10849 master.cc:537] Master@127.10.81.126:37005 shutting down...
01:40:29.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:29.877295 10565 external_mini_cluster.cc:1326] Running /tmp/dist-test-task3Ego3M/build/tsan/bin/kudu
01:40:29.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task3Ego3M/build/tsan/bin/kudu
01:40:29.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-1/wal
01:40:29.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-1/data
01:40:29.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:40:29.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:40:29.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:40:29.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:40:29.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:40:29.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:40:29.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-1/logs
01:40:29.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-1/data/info.pb
01:40:29.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:40:29.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:40:29.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:40:29.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:40:29.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:40:29.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:40:29.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:40:29.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:40:29.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:40:29.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:40:29.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:40:29.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:40:29.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.81.125:38443
01:40:29.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.81.125
01:40:29.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37035
01:40:29.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/krb5kdc/kudu_127.10.81.125.keytab
01:40:29.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.10.81.125
01:40:29.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:40:29.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
01:40:29.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
01:40:29.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.81.84:36591
01:40:29.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:40:29.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:40:29.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:40:29.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:40:29.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975
01:40:29.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:40:29.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42747/jwks.json
01:40:29.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
01:40:29.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:29.930965 10849 master.cc:552] Master@127.10.81.126:37005 shutdown complete.
01:40:29.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
01:40:30.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:30.156770 10922 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:40:30.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:30.157318 10922 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:40:30.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:30.157923 10922 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:40:30.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:30.185782 10922 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
01:40:30.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:30.186003 10922 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:40:30.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:30.186354 10922 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:40:30.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:30.186612 10922 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:40:30.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:30.186877 10922 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:40:30.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:30.187132 10922 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:40:30.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:30.187361 10922 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42747/jwks.json
01:40:30.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:30.220288 10922 master_runner.cc:386] Master server non-default flags:
01:40:30.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:40:30.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.81.84:36591
01:40:30.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:40:30.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:40:30.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-1/data
01:40:30.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-1/wal
01:40:30.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
01:40:30.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:40:30.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975
01:40:30.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/krb5kdc/kudu_127.10.81.125.keytab
01:40:30.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.10.81.125
01:40:30.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:40:30.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:40:30.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:40:30.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.81.125:38443
01:40:30.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:40:30.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:40:30.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:40:30.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42747/jwks.json
01:40:30.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:40:30.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
01:40:30.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
01:40:30.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:40:30.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:40:30.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-1/data/info.pb
01:40:30.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.81.125
01:40:30.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37035
01:40:30.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:40:30.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:40:30.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:40:30.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:40:30.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:40:30.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-1/logs
01:40:30.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:40:30.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:40:30.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:40:30.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:40:30.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:40:30.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:40:30.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:40:30.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 00:52:14 UTC on ccycloud.quasar-sdbdje.root.comops.site
01:40:30.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457837
01:40:30.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:40:30.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:30.221582 10922 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:40:30.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:30.222795 10922 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:40:30.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:30.240399 10930 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:40:31.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:30.240795 10932 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:40:31.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:30.241780 10929 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:40:31.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:30.241926 10922 server_base.cc:1034] running on GCE node
01:40:31.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:31.346037 10922 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:40:31.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:31.348278 10922 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:40:31.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:31.349684 10922 hybrid_clock.cc:648] HybridClock initialized: now 1782006031349650 us; error 40 us; skew 500 ppm
01:40:31.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:31.350380 10922 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:40:31.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:40:31 dist-test-slave-20xp krb5kdc[10578](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006031, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.81.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:40:31.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:31.358943 10922 init.cc:377] Logged in from keytab as kudu/127.10.81.125@KRBTEST.COM (short username kudu)
01:40:31.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:31.364231 10922 webserver.cc:460] Webserver started at http://127.10.81.125:37035/ using document root <none> and password file <none>
01:40:31.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:31.365098 10922 fs_manager.cc:359] Metadata directory not provided
01:40:31.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:31.365408 10922 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:40:31.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:31.372505 10922 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.004s sys 0.000s
01:40:31.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:31.376883 10941 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:40:31.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:31.377950 10922 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
01:40:31.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:31.378273 10922 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-1/data,/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-1/wal
01:40:31.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "29fc73334b354730b94f4afda636e1a5"
01:40:31.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:40:26 on dist-test-slave-20xp"
01:40:31.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:31.380483 10922 fs_report.cc:383] FS layout report
01:40:31.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:40:31.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-1/wal
01:40:31.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-1/wal
01:40:31.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-1/data/data
01:40:31.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:40:31.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:40:31.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:40:31.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:40:31.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:40:31.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:40:31.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:40:31.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:40:31.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:40:31.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:40:31.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:40:31.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:31.435134 10922 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:40:31.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:31.440397 10922 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:40:31.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:31.440805 10922 kserver.cc:163] Server-wide thread pool size limit: 3276
01:40:31.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:31.509066 10922 rpc_server.cc:307] RPC server started. Bound to: 127.10.81.125:38443
01:40:31.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:31.509174 10992 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.81.125:38443 every 8 connection(s)
01:40:31.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:31.511766 10922 server_base.cc:1166] Dumped server information to /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-1/data/info.pb
01:40:31.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:31.520689 10993 sys_catalog.cc:263] Verifying existing consensus state
01:40:31.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260621 01:40:31.521250 10993 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
01:40:31.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:31.521590 10922 master.cc:537] Master@127.10.81.125:38443 shutting down...
01:40:31.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:31.521873 10565 external_mini_cluster.cc:1388] Started /tmp/dist-test-task3Ego3M/build/tsan/bin/kudu as pid 10922
01:40:31.523 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:40:31.523 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.10.81.124:43975
01:40:31.523 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
01:40:31.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:31.525290 10565 external_mini_cluster.cc:1326] Running /tmp/dist-test-task3Ego3M/build/tsan/bin/kudu
01:40:31.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task3Ego3M/build/tsan/bin/kudu
01:40:31.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-2/wal
01:40:31.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-2/data
01:40:31.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:40:31.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:40:31.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:40:31.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:40:31.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:40:31.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:40:31.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-2/logs
01:40:31.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-2/data/info.pb
01:40:31.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:40:31.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:40:31.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:40:31.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:40:31.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:40:31.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:40:31.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:40:31.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:40:31.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:40:31.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:40:31.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:40:31.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:40:31.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.81.124:43975
01:40:31.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.81.124
01:40:31.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41835
01:40:31.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/krb5kdc/kudu_127.10.81.124.keytab
01:40:31.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.10.81.124
01:40:31.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:40:31.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
01:40:31.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
01:40:31.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.81.84:36591
01:40:31.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:40:31.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:40:31.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:40:31.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:40:31.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975
01:40:31.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:40:31.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42747/jwks.json
01:40:31.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
01:40:31.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:31.573622 10922 master.cc:552] Master@127.10.81.125:38443 shutdown complete.
01:40:31.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
01:40:31.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:31.800333 10995 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:40:31.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:31.800805 10995 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:40:31.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:31.801441 10995 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:40:31.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:31.829212 10995 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
01:40:31.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:31.829423 10995 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:40:31.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:31.829706 10995 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:40:31.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:31.829893 10995 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:40:31.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:31.830075 10995 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:40:31.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:31.830257 10995 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:40:31.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:31.830420 10995 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42747/jwks.json
01:40:31.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:31.861640 10995 master_runner.cc:386] Master server non-default flags:
01:40:31.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:40:31.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.81.84:36591
01:40:31.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:40:31.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:40:31.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-2/data
01:40:31.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-2/wal
01:40:31.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
01:40:31.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:40:31.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975
01:40:31.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/krb5kdc/kudu_127.10.81.124.keytab
01:40:31.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.10.81.124
01:40:31.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:40:31.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:40:31.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:40:31.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.81.124:43975
01:40:31.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:40:31.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:40:31.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:40:31.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42747/jwks.json
01:40:31.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:40:31.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
01:40:31.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
01:40:31.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:40:31.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:40:31.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-2/data/info.pb
01:40:31.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.81.124
01:40:31.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41835
01:40:31.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:40:31.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:40:31.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:40:31.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:40:31.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:40:31.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-2/logs
01:40:31.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:40:31.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:40:31.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:40:31.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:40:31.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:40:31.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:40:31.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:40:31.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 00:52:14 UTC on ccycloud.quasar-sdbdje.root.comops.site
01:40:31.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457837
01:40:31.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:40:31.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:31.862658 10995 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:40:31.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:31.863742 10995 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:40:31.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:31.878691 11004 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:40:32.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:31.879061 11006 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:40:32.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:31.878702 11003 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:40:32.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:31.880865 10995 server_base.cc:1034] running on GCE node
01:40:32.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:32.973111 10995 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:40:32.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:32.976155 10995 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:40:32.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:32.977492 10995 hybrid_clock.cc:648] HybridClock initialized: now 1782006032977465 us; error 43 us; skew 500 ppm
01:40:32.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:32.978541 10995 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:40:32.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:40:32 dist-test-slave-20xp krb5kdc[10578](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006032, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.81.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:40:32.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:32.984043 10995 init.cc:377] Logged in from keytab as kudu/127.10.81.124@KRBTEST.COM (short username kudu)
01:40:32.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:32.987965 10995 webserver.cc:460] Webserver started at http://127.10.81.124:41835/ using document root <none> and password file <none>
01:40:32.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:32.988831 10995 fs_manager.cc:359] Metadata directory not provided
01:40:32.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:32.989063 10995 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:40:32.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:32.996107 10995 fs_manager.cc:709] Time spent opening directory manager: real 0.004s user 0.003s sys 0.001s
01:40:33.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:32.999986 11014 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:40:33.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:33.000820 10995 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.002s
01:40:33.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:33.001109 10995 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-2/data,/tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-2/wal
01:40:33.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7c78c4f5682e4bfd96481a43439d7e92"
01:40:33.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 01:40:27 on dist-test-slave-20xp"
01:40:33.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:33.003211 10995 fs_report.cc:383] FS layout report
01:40:33.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:40:33.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-2/wal
01:40:33.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-2/wal
01:40:33.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-2/data/data
01:40:33.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:40:33.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:40:33.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:40:33.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:40:33.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:40:33.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:40:33.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:40:33.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:40:33.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:40:33.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:40:33.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:40:33.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:33.030504 10995 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:40:33.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:33.035386 10995 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:40:33.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:33.035820 10995 kserver.cc:163] Server-wide thread pool size limit: 3276
01:40:33.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:33.107810 10995 rpc_server.cc:307] RPC server started. Bound to: 127.10.81.124:43975
01:40:33.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:33.107923 11065 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.81.124:43975 every 8 connection(s)
01:40:33.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:33.110875 10995 server_base.cc:1166] Dumped server information to /tmp/dist-test-task3Ego3M/test-tmp/mini-kudu-cluster2092467165475114736/master-2/data/info.pb
01:40:33.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:33.119078 10565 external_mini_cluster.cc:1388] Started /tmp/dist-test-task3Ego3M/build/tsan/bin/kudu as pid 10995
01:40:33.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:33.119726 11066 sys_catalog.cc:263] Verifying existing consensus state
01:40:33.120 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:40:33.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:33.124326 11066 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92: Bootstrap starting.
01:40:33.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:33.156138 11066 log.cc:824] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92: Log is configured to *not* fsync() on all Append() calls
01:40:33.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:33.160367 11066 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
01:40:33.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:33.161012 11066 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92: Bootstrap complete.
01:40:33.161 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.10.81.126:37005(127.10.81.126:37005)] connecting to peer
01:40:33.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:33.184691 11066 raft_consensus.cc:348] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02bdac1791354adfb4823bed9e4e2e56" member_type: VOTER last_known_addr { host: "127.10.81.126" port: 37005 } } peers { permanent_uuid: "29fc73334b354730b94f4afda636e1a5" member_type: VOTER last_known_addr { host: "127.10.81.125" port: 38443 } } peers { permanent_uuid: "7c78c4f5682e4bfd96481a43439d7e92" member_type: VOTER last_known_addr { host: "127.10.81.124" port: 43975 } }
01:40:33.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:33.186844 11066 raft_consensus.cc:374] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
01:40:33.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:33.187103 11066 raft_consensus.cc:729] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7c78c4f5682e4bfd96481a43439d7e92, State: Initialized, Role: FOLLOWER
01:40:33.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:33.188129 11066 consensus_queue.cc:260] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02bdac1791354adfb4823bed9e4e2e56" member_type: VOTER last_known_addr { host: "127.10.81.126" port: 37005 } } peers { permanent_uuid: "29fc73334b354730b94f4afda636e1a5" member_type: VOTER last_known_addr { host: "127.10.81.125" port: 38443 } } peers { permanent_uuid: "7c78c4f5682e4bfd96481a43439d7e92" member_type: VOTER last_known_addr { host: "127.10.81.124" port: 43975 } }
01:40:33.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:33.191682 11070 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02bdac1791354adfb4823bed9e4e2e56" member_type: VOTER last_known_addr { host: "127.10.81.126" port: 37005 } } peers { permanent_uuid: "29fc73334b354730b94f4afda636e1a5" member_type: VOTER last_known_addr { host: "127.10.81.125" port: 38443 } } peers { permanent_uuid: "7c78c4f5682e4bfd96481a43439d7e92" member_type: VOTER last_known_addr { host: "127.10.81.124" port: 43975 } } }
01:40:33.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:33.193092 11070 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92 [sys.catalog]: This master's current role is: FOLLOWER
01:40:33.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:33.192839 11066 sys_catalog.cc:564] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92 [sys.catalog]: configured and running, proceeding with master startup.
01:40:33.214 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.10.81.125:38443(127.10.81.125:38443)] connecting to peer
01:40:33.217 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.10.81.124:43975(127.10.81.124:43975)] connecting to peer
01:40:33.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:33.250909 11083 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:40:33.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:33.251253 11083 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:40:33.272 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:40:33.273 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.81.125:38443(127.10.81.125:38443)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:40:33.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:33.245807 10995 master_runner.cc:186] Error getting master registration for 127.10.81.126:37005: Network error: Client connection negotiation failed: client connection to 127.10.81.126:37005: connect: Connection refused (error 111),
01:40:33.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:33.273818 10995 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:40:33.274 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.81.125:38443: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:40:33.278 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:40:33.279 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.81.126:37005(127.10.81.126:37005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:40:33.280 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.81.126:37005: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:40:33.280 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.10.81.124:43975(127.10.81.124:43975)] Successfully connected to peer
01:40:33.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:33.290992 11020 tablet.cc:2369] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92: Can't schedule compaction. Clean time has not been advanced past its initial value.
01:40:33.344 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
01:40:33.351 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.10.81.124
01:40:33.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:33.396726 11070 raft_consensus.cc:482] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:40:33.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:33.397197 11070 raft_consensus.cc:504] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02bdac1791354adfb4823bed9e4e2e56" member_type: VOTER last_known_addr { host: "127.10.81.126" port: 37005 } } peers { permanent_uuid: "29fc73334b354730b94f4afda636e1a5" member_type: VOTER last_known_addr { host: "127.10.81.125" port: 38443 } } peers { permanent_uuid: "7c78c4f5682e4bfd96481a43439d7e92" member_type: VOTER last_known_addr { host: "127.10.81.124" port: 43975 } }
01:40:33.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:33.403609 11016 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.81.125:38443: connect: Connection refused (error 111)
01:40:33.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:33.406476 11070 leader_election.cc:288] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 02bdac1791354adfb4823bed9e4e2e56 (127.10.81.126:37005), 29fc73334b354730b94f4afda636e1a5 (127.10.81.125:38443)
01:40:33.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:33.408208 11016 leader_election.cc:334] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 29fc73334b354730b94f4afda636e1a5 (127.10.81.125:38443): Network error: Client connection negotiation failed: client connection to 127.10.81.125:38443: connect: Connection refused (error 111)
01:40:33.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:33.419777 11015 leader_election.cc:334] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 02bdac1791354adfb4823bed9e4e2e56 (127.10.81.126:37005): Network error: Client connection negotiation failed: client connection to 127.10.81.126:37005: connect: Connection refused (error 111)
01:40:33.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:33.420140 11015 leader_election.cc:302] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 7c78c4f5682e4bfd96481a43439d7e92; no voters: 02bdac1791354adfb4823bed9e4e2e56, 29fc73334b354730b94f4afda636e1a5
01:40:33.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:33.420933 11070 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
01:40:33.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:40:33 dist-test-slave-20xp krb5kdc[10578](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006028, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.10.81.124@KRBTEST.COM
01:40:33.584 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xcdb4d435, L:/127.0.0.1:34190 - R:/127.10.81.124:43975] using SASL/GSSAPI
01:40:33.598 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975; will retry
01:40:33.605 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.81.126:37005(127.10.81.126:37005)] connecting to peer
01:40:33.606 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.81.125:38443(127.10.81.125:38443)] connecting to peer
01:40:33.611 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:40:33.611 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.81.126:37005(127.10.81.126:37005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:40:33.612 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.81.126:37005: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:40:33.613 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:40:33.613 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.81.125:38443(127.10.81.125:38443)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:40:33.613 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.81.125:38443: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:40:33.615 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975; will retry
01:40:33.624 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.81.126:37005(127.10.81.126:37005)] connecting to peer
01:40:33.625 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.81.125:38443(127.10.81.125:38443)] connecting to peer
01:40:33.626 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:40:33.626 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.81.126:37005(127.10.81.126:37005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:40:33.626 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.81.126:37005: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:40:33.628 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:40:33.628 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.81.125:38443(127.10.81.125:38443)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:40:33.629 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.81.125:38443: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:40:33.630 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975; will retry
01:40:33.643 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.81.126:37005(127.10.81.126:37005)] connecting to peer
01:40:33.644 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.81.125:38443(127.10.81.125:38443)] connecting to peer
01:40:33.645 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:40:33.645 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.81.126:37005(127.10.81.126:37005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:40:33.646 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.81.126:37005: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:40:33.647 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:40:33.647 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.81.125:38443(127.10.81.125:38443)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:40:33.648 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.81.125:38443: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:40:33.650 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975; will retry
01:40:33.663 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.81.126:37005(127.10.81.126:37005)] connecting to peer
01:40:33.664 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.81.125:38443(127.10.81.125:38443)] connecting to peer
01:40:33.665 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:40:33.665 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.81.126:37005(127.10.81.126:37005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:40:33.666 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.81.126:37005: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:40:33.670 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:40:33.670 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.81.125:38443(127.10.81.125:38443)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:40:33.671 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.81.125:38443: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:40:33.674 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975; will retry
01:40:33.704 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.81.126:37005(127.10.81.126:37005)] connecting to peer
01:40:33.705 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.81.125:38443(127.10.81.125:38443)] connecting to peer
01:40:33.706 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:40:33.706 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.81.126:37005(127.10.81.126:37005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:40:33.706 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:40:33.707 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.81.125:38443(127.10.81.125:38443)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:40:33.707 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.81.126:37005: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:40:33.708 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.81.125:38443: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:40:33.712 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975; will retry
01:40:33.764 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.81.126:37005(127.10.81.126:37005)] connecting to peer
01:40:33.766 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.81.125:38443(127.10.81.125:38443)] connecting to peer
01:40:33.766 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:40:33.767 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.81.126:37005(127.10.81.126:37005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:40:33.768 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.81.126:37005: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:40:33.769 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:40:33.769 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.81.125:38443(127.10.81.125:38443)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:40:33.770 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.81.125:38443: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:40:33.772 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975; will retry
01:40:33.884 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.81.126:37005(127.10.81.126:37005)] connecting to peer
01:40:33.886 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.81.125:38443(127.10.81.125:38443)] connecting to peer
01:40:33.887 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:40:33.887 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.81.126:37005(127.10.81.126:37005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:40:33.888 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.81.126:37005: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:40:33.889 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:40:33.890 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.81.125:38443(127.10.81.125:38443)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:40:33.890 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.81.125:38443: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:40:33.892 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975; will retry
01:40:34.043 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.81.126:37005(127.10.81.126:37005)] connecting to peer
01:40:34.045 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.81.125:38443(127.10.81.125:38443)] connecting to peer
01:40:34.045 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:40:34.045 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.81.126:37005(127.10.81.126:37005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:40:34.046 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.81.126:37005: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:40:34.049 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:40:34.050 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.81.125:38443(127.10.81.125:38443)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:40:34.050 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.81.125:38443: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:40:34.052 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975; will retry
01:40:34.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:34.255324 11083 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:40:34.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:34.255689 11083 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:40:34.503 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.81.126:37005(127.10.81.126:37005)] connecting to peer
01:40:34.505 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:40:34.505 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.81.126:37005(127.10.81.126:37005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:40:34.506 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.81.125:38443(127.10.81.125:38443)] connecting to peer
01:40:34.506 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.81.126:37005: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:40:34.510 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:40:34.510 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.81.125:38443(127.10.81.125:38443)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:40:34.512 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.81.125:38443: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:40:34.514 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975; will retry
01:40:34.544 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.81.126:37005(127.10.81.126:37005)] connecting to peer
01:40:34.545 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:40:34.545 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.81.126:37005(127.10.81.126:37005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:40:34.545 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.81.126:37005: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:40:34.545 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.81.125:38443(127.10.81.125:38443)] connecting to peer
01:40:34.547 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:40:34.547 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.81.125:38443(127.10.81.125:38443)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:40:34.547 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.81.125:38443: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:40:34.549 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975; will retry
01:40:35.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:35.258145 11083 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:40:35.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:35.258349 11083 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:40:35.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:35.573030 11090 raft_consensus.cc:482] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:40:35.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:35.573324 11090 raft_consensus.cc:504] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02bdac1791354adfb4823bed9e4e2e56" member_type: VOTER last_known_addr { host: "127.10.81.126" port: 37005 } } peers { permanent_uuid: "29fc73334b354730b94f4afda636e1a5" member_type: VOTER last_known_addr { host: "127.10.81.125" port: 38443 } } peers { permanent_uuid: "7c78c4f5682e4bfd96481a43439d7e92" member_type: VOTER last_known_addr { host: "127.10.81.124" port: 43975 } }
01:40:35.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:35.574707 11090 leader_election.cc:288] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 02bdac1791354adfb4823bed9e4e2e56 (127.10.81.126:37005), 29fc73334b354730b94f4afda636e1a5 (127.10.81.125:38443)
01:40:35.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:35.580504 11015 leader_election.cc:334] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 02bdac1791354adfb4823bed9e4e2e56 (127.10.81.126:37005): Network error: Client connection negotiation failed: client connection to 127.10.81.126:37005: connect: Connection refused (error 111)
01:40:35.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:35.580888 11016 leader_election.cc:334] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 29fc73334b354730b94f4afda636e1a5 (127.10.81.125:38443): Network error: Client connection negotiation failed: client connection to 127.10.81.125:38443: connect: Connection refused (error 111)
01:40:35.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:35.581194 11016 leader_election.cc:302] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 7c78c4f5682e4bfd96481a43439d7e92; no voters: 02bdac1791354adfb4823bed9e4e2e56, 29fc73334b354730b94f4afda636e1a5
01:40:35.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:35.581681 11090 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
01:40:36.104 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.81.126:37005(127.10.81.126:37005)] connecting to peer
01:40:36.106 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.81.125:38443(127.10.81.125:38443)] connecting to peer
01:40:36.106 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:40:36.107 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.81.126:37005(127.10.81.126:37005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:40:36.107 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.81.126:37005: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:40:36.108 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:40:36.109 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.81.125:38443(127.10.81.125:38443)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:40:36.109 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.81.125:38443: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:40:36.111 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975; will retry
01:40:36.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:36.260986 11083 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:40:36.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:36.261224 11083 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:40:37.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:37.265254 11083 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:40:37.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:37.266044 11083 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:40:37.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:37.621779 11097 raft_consensus.cc:482] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:40:37.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:37.622097 11097 raft_consensus.cc:504] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02bdac1791354adfb4823bed9e4e2e56" member_type: VOTER last_known_addr { host: "127.10.81.126" port: 37005 } } peers { permanent_uuid: "29fc73334b354730b94f4afda636e1a5" member_type: VOTER last_known_addr { host: "127.10.81.125" port: 38443 } } peers { permanent_uuid: "7c78c4f5682e4bfd96481a43439d7e92" member_type: VOTER last_known_addr { host: "127.10.81.124" port: 43975 } }
01:40:37.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:37.623278 11097 leader_election.cc:288] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 02bdac1791354adfb4823bed9e4e2e56 (127.10.81.126:37005), 29fc73334b354730b94f4afda636e1a5 (127.10.81.125:38443)
01:40:37.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:37.628101 11016 leader_election.cc:334] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 29fc73334b354730b94f4afda636e1a5 (127.10.81.125:38443): Network error: Client connection negotiation failed: client connection to 127.10.81.125:38443: connect: Connection refused (error 111)
01:40:37.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:37.628553 11015 leader_election.cc:334] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 02bdac1791354adfb4823bed9e4e2e56 (127.10.81.126:37005): Network error: Client connection negotiation failed: client connection to 127.10.81.126:37005: connect: Connection refused (error 111)
01:40:37.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:37.628925 11015 leader_election.cc:302] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 7c78c4f5682e4bfd96481a43439d7e92; no voters: 02bdac1791354adfb4823bed9e4e2e56, 29fc73334b354730b94f4afda636e1a5
01:40:37.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:37.629494 11097 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
01:40:38.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:38.264672 10071 debug-util.cc:398] Leaking SignalData structure 0x7b08000bf3e0 after lost signal to thread 10003
01:40:38.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:38.265321 10071 debug-util.cc:398] Leaking SignalData structure 0x7b080009c220 after lost signal to thread 10074
01:40:38.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:38.270351 11083 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:40:38.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:38.270699 11083 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:40:38.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:38.276357 10995 master_runner.cc:186] Error getting master registration for 127.10.81.126:37005: Network error: Client connection negotiation failed: client connection to 127.10.81.126:37005: connect: Connection refused (error 111),
01:40:38.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:38.276753 10995 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:40:39.264 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.81.126:37005(127.10.81.126:37005)] connecting to peer
01:40:39.265 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.81.125:38443(127.10.81.125:38443)] connecting to peer
01:40:39.266 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:40:39.266 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.81.126:37005(127.10.81.126:37005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:40:39.266 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.81.126:37005: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:40:39.268 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:40:39.268 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.81.125:38443(127.10.81.125:38443)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:40:39.269 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.81.125:38443: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:40:39.270 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975; will retry
01:40:39.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:39.274607 11083 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:40:39.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:39.274888 11083 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:40:39.524 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.81.126:37005(127.10.81.126:37005)] connecting to peer
01:40:39.526 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.81.125:38443(127.10.81.125:38443)] connecting to peer
01:40:39.526 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:40:39.526 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.81.126:37005(127.10.81.126:37005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:40:39.526 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.81.126:37005: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:40:39.528 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:40:39.529 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.81.125:38443(127.10.81.125:38443)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:40:39.529 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.81.125:38443: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:40:39.531 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975; will retry
01:40:39.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:39.827554 11105 raft_consensus.cc:482] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:40:39.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:39.827834 11105 raft_consensus.cc:504] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02bdac1791354adfb4823bed9e4e2e56" member_type: VOTER last_known_addr { host: "127.10.81.126" port: 37005 } } peers { permanent_uuid: "29fc73334b354730b94f4afda636e1a5" member_type: VOTER last_known_addr { host: "127.10.81.125" port: 38443 } } peers { permanent_uuid: "7c78c4f5682e4bfd96481a43439d7e92" member_type: VOTER last_known_addr { host: "127.10.81.124" port: 43975 } }
01:40:39.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:39.829114 11105 leader_election.cc:288] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 02bdac1791354adfb4823bed9e4e2e56 (127.10.81.126:37005), 29fc73334b354730b94f4afda636e1a5 (127.10.81.125:38443)
01:40:39.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:39.830688 11015 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.81.126:37005: connect: Connection refused (error 111) [suppressed 5 similar messages]
01:40:39.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:39.833027 11015 leader_election.cc:334] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 02bdac1791354adfb4823bed9e4e2e56 (127.10.81.126:37005): Network error: Client connection negotiation failed: client connection to 127.10.81.126:37005: connect: Connection refused (error 111)
01:40:39.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:39.833384 11016 leader_election.cc:334] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 29fc73334b354730b94f4afda636e1a5 (127.10.81.125:38443): Network error: Client connection negotiation failed: client connection to 127.10.81.125:38443: connect: Connection refused (error 111)
01:40:39.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:39.833660 11016 leader_election.cc:302] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 7c78c4f5682e4bfd96481a43439d7e92; no voters: 02bdac1791354adfb4823bed9e4e2e56, 29fc73334b354730b94f4afda636e1a5
01:40:39.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:39.834062 11105 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
01:40:40.144 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.81.126:37005(127.10.81.126:37005)] connecting to peer
01:40:40.145 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.81.125:38443(127.10.81.125:38443)] connecting to peer
01:40:40.146 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:40:40.146 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.81.126:37005(127.10.81.126:37005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:40:40.146 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.81.126:37005: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:40:40.147 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:40:40.147 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.81.125:38443(127.10.81.125:38443)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:40:40.148 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.81.125:38443: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:40:40.150 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975; will retry
01:40:40.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:40.277331 11083 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:40:40.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:40.277592 11083 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:40:41.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:41.280081 11083 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:40:41.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:41.280290 11083 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:40:41.804 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.81.126:37005(127.10.81.126:37005)] connecting to peer
01:40:41.805 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.81.125:38443(127.10.81.125:38443)] connecting to peer
01:40:41.806 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:40:41.806 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.81.126:37005(127.10.81.126:37005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:40:41.807 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.81.126:37005: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:40:41.808 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:40:41.809 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.81.125:38443(127.10.81.125:38443)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:40:41.809 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.81.125:38443: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:40:41.811 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975; will retry
01:40:42.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:42.282934 11083 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:40:42.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:42.283253 11083 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:40:42.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:40:42 dist-test-slave-20xp krb5kdc[10578](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006042, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.81.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:40:42.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:42.987289 11010 init.cc:295] Successfully reacquired a new kerberos TGT
01:40:43.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:43.279479 10995 master_runner.cc:186] Error getting master registration for 127.10.81.126:37005: Network error: Client connection negotiation failed: client connection to 127.10.81.126:37005: connect: Connection refused (error 111),
01:40:43.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:43.279845 10995 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:40:43.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:43.285588 11083 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:40:43.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:43.285771 11083 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:40:44.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:44.288053 11083 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:40:44.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:44.288262 11083 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:40:45.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:45.290760 11083 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:40:45.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:45.291026 11083 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:40:45.744 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.81.126:37005(127.10.81.126:37005)] connecting to peer
01:40:45.745 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.81.125:38443(127.10.81.125:38443)] connecting to peer
01:40:45.746 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:40:45.746 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.81.126:37005(127.10.81.126:37005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:40:45.746 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.81.126:37005: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:40:45.747 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:40:45.748 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.81.125:38443(127.10.81.125:38443)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:40:45.748 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.81.125:38443: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:40:45.749 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975; will retry
01:40:46.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:46.098086 11118 raft_consensus.cc:482] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:40:46.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:46.098547 11118 raft_consensus.cc:504] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02bdac1791354adfb4823bed9e4e2e56" member_type: VOTER last_known_addr { host: "127.10.81.126" port: 37005 } } peers { permanent_uuid: "29fc73334b354730b94f4afda636e1a5" member_type: VOTER last_known_addr { host: "127.10.81.125" port: 38443 } } peers { permanent_uuid: "7c78c4f5682e4bfd96481a43439d7e92" member_type: VOTER last_known_addr { host: "127.10.81.124" port: 43975 } }
01:40:46.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:46.100543 11118 leader_election.cc:288] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 02bdac1791354adfb4823bed9e4e2e56 (127.10.81.126:37005), 29fc73334b354730b94f4afda636e1a5 (127.10.81.125:38443)
01:40:46.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:46.102664 11015 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.81.126:37005: connect: Connection refused (error 111) [suppressed 1 similar messages]
01:40:46.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:46.105268 11016 leader_election.cc:334] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 29fc73334b354730b94f4afda636e1a5 (127.10.81.125:38443): Network error: Client connection negotiation failed: client connection to 127.10.81.125:38443: connect: Connection refused (error 111)
01:40:46.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:46.105691 11015 leader_election.cc:334] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 02bdac1791354adfb4823bed9e4e2e56 (127.10.81.126:37005): Network error: Client connection negotiation failed: client connection to 127.10.81.126:37005: connect: Connection refused (error 111)
01:40:46.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:46.106029 11015 leader_election.cc:302] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 7c78c4f5682e4bfd96481a43439d7e92; no voters: 02bdac1791354adfb4823bed9e4e2e56, 29fc73334b354730b94f4afda636e1a5
01:40:46.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:46.106745 11118 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
01:40:46.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:46.293650 11083 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:40:46.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:46.293936 11083 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:40:47.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:47.296387 11083 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:40:47.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:47.296590 11083 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:40:48.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:48.281977 10995 master_runner.cc:186] Error getting master registration for 127.10.81.126:37005: Network error: Client connection negotiation failed: client connection to 127.10.81.126:37005: connect: Connection refused (error 111),
01:40:48.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:48.282305 10995 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:40:48.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:48.298871 11083 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:40:48.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:48.299083 11083 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:40:48.904 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.81.126:37005(127.10.81.126:37005)] connecting to peer
01:40:48.905 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.81.125:38443(127.10.81.125:38443)] connecting to peer
01:40:48.906 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:40:48.906 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.81.126:37005(127.10.81.126:37005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:40:48.906 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.81.126:37005: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:40:48.907 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:40:48.907 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.81.125:38443(127.10.81.125:38443)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:40:48.907 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.81.125:38443: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:40:48.910 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975; will retry
01:40:49.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:49.301602 11083 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:40:49.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:49.301805 11083 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:40:50.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:50.304090 11083 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:40:50.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:50.304363 11083 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:40:50.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:50.461161 11127 raft_consensus.cc:482] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:40:50.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:50.461496 11127 raft_consensus.cc:504] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02bdac1791354adfb4823bed9e4e2e56" member_type: VOTER last_known_addr { host: "127.10.81.126" port: 37005 } } peers { permanent_uuid: "29fc73334b354730b94f4afda636e1a5" member_type: VOTER last_known_addr { host: "127.10.81.125" port: 38443 } } peers { permanent_uuid: "7c78c4f5682e4bfd96481a43439d7e92" member_type: VOTER last_known_addr { host: "127.10.81.124" port: 43975 } }
01:40:50.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:50.463078 11127 leader_election.cc:288] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 02bdac1791354adfb4823bed9e4e2e56 (127.10.81.126:37005), 29fc73334b354730b94f4afda636e1a5 (127.10.81.125:38443)
01:40:50.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:50.467603 11015 leader_election.cc:334] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 02bdac1791354adfb4823bed9e4e2e56 (127.10.81.126:37005): Network error: Client connection negotiation failed: client connection to 127.10.81.126:37005: connect: Connection refused (error 111)
01:40:50.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:50.468168 11016 leader_election.cc:334] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 29fc73334b354730b94f4afda636e1a5 (127.10.81.125:38443): Network error: Client connection negotiation failed: client connection to 127.10.81.125:38443: connect: Connection refused (error 111)
01:40:50.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:50.468586 11016 leader_election.cc:302] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 7c78c4f5682e4bfd96481a43439d7e92; no voters: 02bdac1791354adfb4823bed9e4e2e56, 29fc73334b354730b94f4afda636e1a5
01:40:50.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:50.469272 11127 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
01:40:50.904 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.81.126:37005(127.10.81.126:37005)] connecting to peer
01:40:50.905 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.81.125:38443(127.10.81.125:38443)] connecting to peer
01:40:50.906 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:40:50.906 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.81.126:37005(127.10.81.126:37005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:40:50.907 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.81.126:37005: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:40:50.912 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:40:50.912 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.81.125:38443(127.10.81.125:38443)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:40:50.912 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.81.125:38443: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:40:50.912 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975; will retry
01:40:50.984 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.81.126:37005(127.10.81.126:37005)] connecting to peer
01:40:50.985 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.81.125:38443(127.10.81.125:38443)] connecting to peer
01:40:50.985 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:40:50.986 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.81.126:37005(127.10.81.126:37005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:40:50.986 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.81.126:37005: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:40:50.987 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:40:50.987 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.81.125:38443(127.10.81.125:38443)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:40:50.987 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.81.125:38443: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:40:50.988 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975; will retry
01:40:51.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:51.306787 11083 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:40:51.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:51.307015 11083 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:40:52.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:52.309949 11083 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:40:52.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:52.310271 11083 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:40:52.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 01:40:52 dist-test-slave-20xp krb5kdc[10578](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782006052, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.81.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:40:52.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:52.989236 11010 init.cc:295] Successfully reacquired a new kerberos TGT
01:40:53.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:53.285175 10995 master_runner.cc:186] Error getting master registration for 127.10.81.126:37005: Network error: Client connection negotiation failed: client connection to 127.10.81.126:37005: connect: Connection refused (error 111),
01:40:53.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:53.285540 10995 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:40:53.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:53.312852 11083 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:40:53.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:53.313076 11083 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:40:54.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:54.315398 11083 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:40:54.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:54.315697 11083 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:40:55.084 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.81.126:37005(127.10.81.126:37005)] connecting to peer
01:40:55.085 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.81.125:38443(127.10.81.125:38443)] connecting to peer
01:40:55.085 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:40:55.085 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.81.126:37005(127.10.81.126:37005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:40:55.086 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.81.126:37005: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:40:55.087 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:40:55.087 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.81.125:38443(127.10.81.125:38443)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:40:55.087 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.81.125:38443: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:40:55.089 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975; will retry
01:40:55.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:55.317983 11083 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:40:55.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:55.318190 11083 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:40:56.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:56.320912 11083 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:40:56.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:56.321136 11083 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:40:57.164 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.81.126:37005(127.10.81.126:37005)] connecting to peer
01:40:57.165 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.81.125:38443(127.10.81.125:38443)] connecting to peer
01:40:57.166 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:40:57.166 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.81.126:37005(127.10.81.126:37005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:40:57.166 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.81.126:37005: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:40:57.166 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:40:57.167 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.81.125:38443(127.10.81.125:38443)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:40:57.167 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.81.125:38443: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:40:57.168 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975; will retry
01:40:57.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:57.324067 11083 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:40:57.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:57.324282 11083 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:40:58.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:58.287791 10995 master_runner.cc:186] Error getting master registration for 127.10.81.126:37005: Network error: Client connection negotiation failed: client connection to 127.10.81.126:37005: connect: Connection refused (error 111),
01:40:58.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:40:58.288061 10995 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:40:58.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:58.326601 11083 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:40:58.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:58.326822 11083 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:40:59.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:59.329146 11083 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:40:59.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:40:59.329365 11083 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:41:00.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:41:00.332212 11083 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7c78c4f5682e4bfd96481a43439d7e92: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:41:00.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 01:41:00.332435 11083 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:41:00.823 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.81.126:37005(127.10.81.126:37005)] connecting to peer
01:41:00.824 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.81.125:38443(127.10.81.125:38443)] connecting to peer
01:41:00.824 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:41:00.825 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.81.126:37005(127.10.81.126:37005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:41:00.825 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.81.126:37005: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005
01:41:00.826 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:41:00.826 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.81.125:38443(127.10.81.125:38443)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:41:00.826 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.81.125:38443: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
01:41:00.828 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975; will retry
01:41:00.833 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=23, TimeoutTracker(timeout=30000, elapsed=27706), Traces: [0ms] refreshing cache from master, [26ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.126:37005, [88ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.125:38443, [91ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.124:43975, [147ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.125:38443: Network error: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [153ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.126:37005: Network error: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005, [469ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.124:43975: OK, [473ms] delaying RPC due to: Service unavailable: Master config (127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005, [478ms] refreshing cache from master, [479ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.126:37005, [480ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.125:38443, [480ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.124:43975, [485ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.126:37005: Network error: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005, [487ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.125:38443: Network error: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [488ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.124:43975: OK, [489ms] delaying RPC due to: Service unavailable: Master config (127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [498ms] refreshing cache from master, [498ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.126:37005, [499ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.125:38443, [500ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.124:43975, [500ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.126:37005: Network error: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005, [502ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.125:38443: Network error: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [504ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.124:43975: OK, [504ms] delaying RPC due to: Service unavailable: Master config (127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [517ms] refreshing cache from master, [517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.126:37005, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.125:38443, [519ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.124:43975, [519ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.126:37005: Network error: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005, [521ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.125:38443: Network error: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [523ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.124:43975: OK, [524ms] delaying RPC due to: Service unavailable: Master config (127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [537ms] refreshing cache from master, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.126:37005, [538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.125:38443, [538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.124:43975, [539ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.126:37005: Network error: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005, [544ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.125:38443: Network error: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [546ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.124:43975: OK, [548ms] delaying RPC due to: Service unavailable: Master config (127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [577ms] refreshing cache from master, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.126:37005, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.125:38443, [580ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.124:43975, [580ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.126:37005: Network error: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005, [581ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.125:38443: Network error: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [585ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.124:43975: OK, [586ms] delaying RPC due to: Service unavailable: Master config (127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [638ms] refreshing cache from master, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.126:37005, [640ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.125:38443, [641ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.124:43975, [642ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.126:37005: Network error: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005, [644ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.125:38443: Network error: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [645ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.124:43975: OK, [646ms] delaying RPC due to: Service unavailable: Master config (127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [758ms] refreshing cache from master, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.126:37005, [760ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.125:38443, [761ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.124:43975, [761ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.126:37005: Network error: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005, [764ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.125:38443: Network error: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [765ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.124:43975: OK, [766ms] delaying RPC due to: Service unavailable: Master config (127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [917ms] refreshing cache from master, [917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.126:37005, [918ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.125:38443, [919ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.124:43975, [920ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.126:37005: Network error: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005, [924ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.125:38443: Network error: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [925ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.124:43975: OK, [926ms] delaying RPC due to: Service unavailable: Master config (127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [1377ms] refreshing cache from master, [1377ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.126:37005, [1380ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.126:37005: Network error: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005, [1380ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.125:38443, [1381ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.124:43975, [1384ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.125:38443: Network error: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [1387ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.124:43975: OK, [1388ms] delaying RPC due to: Service unavailable: Master config (127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [1417ms] refreshing cache from master, [1417ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.126:37005, [1419ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.126:37005: Network error: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005, [1419ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.125:38443, [1420ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.124:43975, [1421ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.125:38443: Network error: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [1423ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.124:43975: OK, [1424ms] delaying RPC due to: Service unavailable: Master config (127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [2977ms] refreshing cache from master, [2978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.126:37005, [2979ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.125:38443, [2981ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.124:43975, [2981ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.126:37005: Network error: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005, [2983ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.125:38443: Network error: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [2984ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.124:43975: OK, [2985ms] delaying RPC due to: Service unavailable: Master config (127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [6138ms] refreshing cache from master, [6138ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.126:37005, [6139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.125:38443, [6140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.124:43975, [6140ms] too many traces: truncated at 100 traces, deferred=Deferred@505536193(state=PENDING, result=null, callback=(continuation of Deferred@1405071283 after retry RPC after error@665788173) -> (continuation of Deferred@1282917142 after retry RPC after error@1546701711) -> (continuation of Deferred@1918641549 after retry RPC after error@760482748) -> (continuation of Deferred@1607012979 after retry RPC after error@100652726) -> (continuation of Deferred@370998217 after retry RPC after error@462953309) -> (continuation of Deferred@206952663 after retry RPC after error@2003105165) -> (continuation of Deferred@106400283 after retry RPC after error@1565404031) -> (continuation of Deferred@2139079521 after retry RPC after error@803707294) -> (continuation of Deferred@366659883 after retry RPC after error@962792764) -> (continuation of Deferred@948352442 after retry RPC after error@733933661) -> (continuation of Deferred@1760491408 after retry RPC after error@1229262172) -> (continuation of Deferred@1147357867 after retry RPC after error@411264610) -> (continuation of Deferred@122572949 after retry RPC after error@1711313613) -> (continuation of Deferred@946387636 after retry RPC after error@995851181) -> (continuation of Deferred@1323589291 after retry RPC after error@739257152) -> (continuation of Deferred@1372625260 after retry RPC after error@825628392) -> (continuation of Deferred@1573590883 after retry RPC after error@1890352561) -> (continuation of Deferred@635605297 after retry RPC after error@610419251) -> (continuation of Deferred@555170259 after retry RPC after error@843418037) -> (continuation of Deferred@52428245 after retry RPC after error@895112287) -> (continuation of Deferred@756755078 after retry RPC after error@856495793) -> (continuation of Deferred@1325442128 after retry RPC after error@673184067), errback=(continuation of Deferred@1405071283 after retry RPC after error@665788173) -> (continuation of Deferred@1282917142 after retry RPC after error@1546701711) -> (continuation of Deferred@1918641549 after retry RPC after error@760482748) -> (continuation of Deferred@1607012979 after retry RPC after error@100652726) -> (continuation of Deferred@370998217 after retry RPC after error@462953309) -> (continuation of Deferred@206952663 after retry RPC after error@2003105165) -> (continuation of Deferred@106400283 after retry RPC after error@1565404031) -> (continuation of Deferred@2139079521 after retry RPC after error@803707294) -> (continuation of Deferred@366659883 after retry RPC after error@962792764) -> (continuation of Deferred@948352442 after retry RPC after error@733933661) -> (continuation of Deferred@1760491408 after retry RPC after error@1229262172) -> (continuation of Deferred@1147357867 after retry RPC after error@411264610) -> (continuation of Deferred@122572949 after retry RPC after error@1711313613) -> (continuation of Deferred@946387636 after retry RPC after error@995851181) -> (continuation of Deferred@1323589291 after retry RPC after error@739257152) -> (continuation of Deferred@1372625260 after retry RPC after error@825628392) -> (continuation of Deferred@1573590883 after retry RPC after error@1890352561) -> (continuation of Deferred@635605297 after retry RPC after error@610419251) -> (continuation of Deferred@555170259 after retry RPC after error@843418037) -> (continuation of Deferred@52428245 after retry RPC after error@895112287) -> (continuation of Deferred@756755078 after retry RPC after error@856495793) -> (continuation of Deferred@1325442128 after retry RPC after error@673184067)))
01:41:00.838 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.10.81.126:37005(127.10.81.126:37005)] disconnecting while in state TERMINATED
01:41:00.839 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.10.81.125:38443(127.10.81.125:38443)] disconnecting while in state TERMINATED
01:41:00.839 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.10.81.124:43975(127.10.81.124:43975)] disconnecting while in state READY
01:41:00.846 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
01:41:00.847 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
01:41:00.849 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.10.81.124:43975(127.10.81.124:43975)] handling channelInactive
01:41:00.849 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.81.124:43975(127.10.81.124:43975)] cleaning up while in state READY due to: connection closed
01:41:00.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:41:00.852133 10565 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task3Ego3M/build/tsan/bin/kudu with pid 10995
01:41:01.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-21T01:41:01Z chronyd exiting
01:41:02.914 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
01:41:02.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:41:02.915172 9841 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task3Ego3M/build/tsan/bin/kudu with pid 10119
01:41:02.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:41:02.936647 9841 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task3Ego3M/build/tsan/bin/kudu with pid 10272
01:41:02.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:41:02.959832 9841 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task3Ego3M/build/tsan/bin/kudu with pid 10420
01:41:02.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:41:02.985011 9841 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task3Ego3M/build/tsan/bin/kudu with pid 9853
01:41:03.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:41:03.009967 9841 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task3Ego3M/build/tsan/bin/kudu with pid 9923
01:41:03.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 01:41:03.033623 9841 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task3Ego3M/build/tsan/bin/kudu with pid 9998
01:41:04.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-21T01:41:04Z chronyd exiting
01:41:05.094 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=23, TimeoutTracker(timeout=30000, elapsed=27706), Traces: [0ms] refreshing cache from master, [26ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.126:37005, [88ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.125:38443, [91ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.124:43975, [147ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.125:38443: Network error: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [153ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.126:37005: Network error: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005, [469ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.124:43975: OK, [473ms] delaying RPC due to: Service unavailable: Master config (127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005, [478ms] refreshing cache from master, [479ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.126:37005, [480ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.125:38443, [480ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.124:43975, [485ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.126:37005: Network error: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005, [487ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.125:38443: Network error: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [488ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.124:43975: OK, [489ms] delaying RPC due to: Service unavailable: Master config (127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [498ms] refreshing cache from master, [498ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.126:37005, [499ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.125:38443, [500ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.124:43975, [500ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.126:37005: Network error: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005, [502ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.125:38443: Network error: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [504ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.124:43975: OK, [504ms] delaying RPC due to: Service unavailable: Master config (127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [517ms] refreshing cache from master, [517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.126:37005, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.125:38443, [519ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.124:43975, [519ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.126:37005: Network error: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005, [521ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.125:38443: Network error: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [523ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.124:43975: OK, [524ms] delaying RPC due to: Service unavailable: Master config (127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [537ms] refreshing cache from master, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.126:37005, [538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.125:38443, [538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.124:43975, [539ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.126:37005: Network error: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005, [544ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.125:38443: Network error: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [546ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.124:43975: OK, [548ms] delaying RPC due to: Service unavailable: Master config (127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [577ms] refreshing cache from master, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.126:37005, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.125:38443, [580ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.124:43975, [580ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.126:37005: Network error: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005, [581ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.125:38443: Network error: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [585ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.124:43975: OK, [586ms] delaying RPC due to: Service unavailable: Master config (127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [638ms] refreshing cache from master, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.126:37005, [640ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.125:38443, [641ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.124:43975, [642ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.126:37005: Network error: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005, [644ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.125:38443: Network error: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [645ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.124:43975: OK, [646ms] delaying RPC due to: Service unavailable: Master config (127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [758ms] refreshing cache from master, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.126:37005, [760ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.125:38443, [761ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.124:43975, [761ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.126:37005: Network error: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005, [764ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.125:38443: Network error: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [765ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.124:43975: OK, [766ms] delaying RPC due to: Service unavailable: Master config (127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [917ms] refreshing cache from master, [917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.126:37005, [918ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.125:38443, [919ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.124:43975, [920ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.126:37005: Network error: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005, [924ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.125:38443: Network error: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [925ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.124:43975: OK, [926ms] delaying RPC due to: Service unavailable: Master config (127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [1377ms] refreshing cache from master, [1377ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.126:37005, [1380ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.126:37005: Network error: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005, [1380ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.125:38443, [1381ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.124:43975, [1384ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.125:38443: Network error: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [1387ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.124:43975: OK, [1388ms] delaying RPC due to: Service unavailable: Master config (127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [1417ms] refreshing cache from master, [1417ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.126:37005, [1419ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.126:37005: Network error: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005, [1419ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.125:38443, [1420ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.124:43975, [1421ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.125:38443: Network error: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [1423ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.124:43975: OK, [1424ms] delaying RPC due to: Service unavailable: Master config (127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [2977ms] refreshing cache from master, [2978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.126:37005, [2979ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.125:38443, [2981ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.124:43975, [2981ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.126:37005: Network error: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005, [2983ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.125:38443: Network error: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [2984ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.124:43975: OK, [2985ms] delaying RPC due to: Service unavailable: Master config (127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [6138ms] refreshing cache from master, [6138ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.126:37005, [6139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.125:38443, [6140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.124:43975, [6140ms] too many traces: truncated at 100 traces, deferred=Deferred@505536193(state=PENDING, result=null, callback=(continuation of Deferred@1405071283 after retry RPC after error@665788173) -> (continuation of Deferred@1282917142 after retry RPC after error@1546701711) -> (continuation of Deferred@1918641549 after retry RPC after error@760482748) -> (continuation of Deferred@1607012979 after retry RPC after error@100652726) -> (continuation of Deferred@370998217 after retry RPC after error@462953309) -> (continuation of Deferred@206952663 after retry RPC after error@2003105165) -> (continuation of Deferred@106400283 after retry RPC after error@1565404031) -> (continuation of Deferred@2139079521 after retry RPC after error@803707294) -> (continuation of Deferred@366659883 after retry RPC after error@962792764) -> (continuation of Deferred@948352442 after retry RPC after error@733933661) -> (continuation of Deferred@1760491408 after retry RPC after error@1229262172) -> (continuation of Deferred@1147357867 after retry RPC after error@411264610) -> (continuation of Deferred@122572949 after retry RPC after error@1711313613) -> (continuation of Deferred@946387636 after retry RPC after error@995851181) -> (continuation of Deferred@1323589291 after retry RPC after error@739257152) -> (continuation of Deferred@1372625260 after retry RPC after error@825628392) -> (continuation of Deferred@1573590883 after retry RPC after error@1890352561) -> (continuation of Deferred@635605297 after retry RPC after error@610419251) -> (continuation of Deferred@555170259 after retry RPC after error@843418037) -> (continuation of Deferred@52428245 after retry RPC after error@895112287) -> (continuation of Deferred@756755078 after retry RPC after error@856495793) -> (continuation of Deferred@1325442128 after retry RPC after error@673184067), errback=(continuation of Deferred@1405071283 after retry RPC after error@665788173) -> (continuation of Deferred@1282917142 after retry RPC after error@1546701711) -> (continuation of Deferred@1918641549 after retry RPC after error@760482748) -> (continuation of Deferred@1607012979 after retry RPC after error@100652726) -> (continuation of Deferred@370998217 after retry RPC after error@462953309) -> (continuation of Deferred@206952663 after retry RPC after error@2003105165) -> (continuation of Deferred@106400283 after retry RPC after error@1565404031) -> (continuation of Deferred@2139079521 after retry RPC after error@803707294) -> (continuation of Deferred@366659883 after retry RPC after error@962792764) -> (continuation of Deferred@948352442 after retry RPC after error@733933661) -> (continuation of Deferred@1760491408 after retry RPC after error@1229262172) -> (continuation of Deferred@1147357867 after retry RPC after error@411264610) -> (continuation of Deferred@122572949 after retry RPC after error@1711313613) -> (continuation of Deferred@946387636 after retry RPC after error@995851181) -> (continuation of Deferred@1323589291 after retry RPC after error@739257152) -> (continuation of Deferred@1372625260 after retry RPC after error@825628392) -> (continuation of Deferred@1573590883 after retry RPC after error@1890352561) -> (continuation of Deferred@635605297 after retry RPC after error@610419251) -> (continuation of Deferred@555170259 after retry RPC after error@843418037) -> (continuation of Deferred@52428245 after retry RPC after error@895112287) -> (continuation of Deferred@756755078 after retry RPC after error@856495793) -> (continuation of Deferred@1325442128 after retry RPC after error@673184067)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.10.81.125:38443
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=23, TimeoutTracker(timeout=30000, elapsed=27706), Traces: [0ms] refreshing cache from master, [26ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.126:37005, [88ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.125:38443, [91ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.124:43975, [147ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.125:38443: Network error: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [153ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.126:37005: Network error: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005, [469ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.124:43975: OK, [473ms] delaying RPC due to: Service unavailable: Master config (127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005, [478ms] refreshing cache from master, [479ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.126:37005, [480ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.125:38443, [480ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.124:43975, [485ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.126:37005: Network error: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005, [487ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.125:38443: Network error: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [488ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.124:43975: OK, [489ms] delaying RPC due to: Service unavailable: Master config (127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [498ms] refreshing cache from master, [498ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.126:37005, [499ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.125:38443, [500ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.124:43975, [500ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.126:37005: Network error: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005, [502ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.125:38443: Network error: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [504ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.124:43975: OK, [504ms] delaying RPC due to: Service unavailable: Master config (127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [517ms] refreshing cache from master, [517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.126:37005, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.125:38443, [519ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.124:43975, [519ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.126:37005: Network error: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005, [521ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.125:38443: Network error: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [523ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.124:43975: OK, [524ms] delaying RPC due to: Service unavailable: Master config (127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [537ms] refreshing cache from master, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.126:37005, [538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.125:38443, [538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.124:43975, [539ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.126:37005: Network error: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005, [544ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.125:38443: Network error: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [546ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.124:43975: OK, [548ms] delaying RPC due to: Service unavailable: Master config (127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [577ms] refreshing cache from master, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.126:37005, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.125:38443, [580ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.124:43975, [580ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.126:37005: Network error: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005, [581ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.125:38443: Network error: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [585ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.124:43975: OK, [586ms] delaying RPC due to: Service unavailable: Master config (127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [638ms] refreshing cache from master, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.126:37005, [640ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.125:38443, [641ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.124:43975, [642ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.126:37005: Network error: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005, [644ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.125:38443: Network error: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [645ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.124:43975: OK, [646ms] delaying RPC due to: Service unavailable: Master config (127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [758ms] refreshing cache from master, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.126:37005, [760ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.125:38443, [761ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.124:43975, [761ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.126:37005: Network error: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005, [764ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.125:38443: Network error: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [765ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.124:43975: OK, [766ms] delaying RPC due to: Service unavailable: Master config (127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [917ms] refreshing cache from master, [917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.126:37005, [918ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.125:38443, [919ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.124:43975, [920ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.126:37005: Network error: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005, [924ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.125:38443: Network error: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [925ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.124:43975: OK, [926ms] delaying RPC due to: Service unavailable: Master config (127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [1377ms] refreshing cache from master, [1377ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.126:37005, [1380ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.126:37005: Network error: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005, [1380ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.125:38443, [1381ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.124:43975, [1384ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.125:38443: Network error: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [1387ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.124:43975: OK, [1388ms] delaying RPC due to: Service unavailable: Master config (127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [1417ms] refreshing cache from master, [1417ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.126:37005, [1419ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.126:37005: Network error: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005, [1419ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.125:38443, [1420ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.124:43975, [1421ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.125:38443: Network error: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [1423ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.124:43975: OK, [1424ms] delaying RPC due to: Service unavailable: Master config (127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [2977ms] refreshing cache from master, [2978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.126:37005, [2979ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.125:38443, [2981ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.124:43975, [2981ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.126:37005: Network error: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005, [2983ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.125:38443: Network error: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [2984ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.124:43975: OK, [2985ms] delaying RPC due to: Service unavailable: Master config (127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [6138ms] refreshing cache from master, [6138ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.126:37005, [6139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.125:38443, [6140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.124:43975, [6140ms] too many traces: truncated at 100 traces, deferred=Deferred@505536193(state=PENDING, result=null, callback=(continuation of Deferred@1405071283 after retry RPC after error@665788173) -> (continuation of Deferred@1282917142 after retry RPC after error@1546701711) -> (continuation of Deferred@1918641549 after retry RPC after error@760482748) -> (continuation of Deferred@1607012979 after retry RPC after error@100652726) -> (continuation of Deferred@370998217 after retry RPC after error@462953309) -> (continuation of Deferred@206952663 after retry RPC after error@2003105165) -> (continuation of Deferred@106400283 after retry RPC after error@1565404031) -> (continuation of Deferred@2139079521 after retry RPC after error@803707294) -> (continuation of Deferred@366659883 after retry RPC after error@962792764) -> (continuation of Deferred@948352442 after retry RPC after error@733933661) -> (continuation of Deferred@1760491408 after retry RPC after error@1229262172) -> (continuation of Deferred@1147357867 after retry RPC after error@411264610) -> (continuation of Deferred@122572949 after retry RPC after error@1711313613) -> (continuation of Deferred@946387636 after retry RPC after error@995851181) -> (continuation of Deferred@1323589291 after retry RPC after error@739257152) -> (continuation of Deferred@1372625260 after retry RPC after error@825628392) -> (continuation of Deferred@1573590883 after retry RPC after error@1890352561) -> (continuation of Deferred@635605297 after retry RPC after error@610419251) -> (continuation of Deferred@555170259 after retry RPC after error@843418037) -> (continuation of Deferred@52428245 after retry RPC after error@895112287) -> (continuation of Deferred@756755078 after retry RPC after error@856495793) -> (continuation of Deferred@1325442128 after retry RPC after error@673184067), errback=(continuation of Deferred@1405071283 after retry RPC after error@665788173) -> (continuation of Deferred@1282917142 after retry RPC after error@1546701711) -> (continuation of Deferred@1918641549 after retry RPC after error@760482748) -> (continuation of Deferred@1607012979 after retry RPC after error@100652726) -> (continuation of Deferred@370998217 after retry RPC after error@462953309) -> (continuation of Deferred@206952663 after retry RPC after error@2003105165) -> (continuation of Deferred@106400283 after retry RPC after error@1565404031) -> (continuation of Deferred@2139079521 after retry RPC after error@803707294) -> (continuation of Deferred@366659883 after retry RPC after error@962792764) -> (continuation of Deferred@948352442 after retry RPC after error@733933661) -> (continuation of Deferred@1760491408 after retry RPC after error@1229262172) -> (continuation of Deferred@1147357867 after retry RPC after error@411264610) -> (continuation of Deferred@122572949 after retry RPC after error@1711313613) -> (continuation of Deferred@946387636 after retry RPC after error@995851181) -> (continuation of Deferred@1323589291 after retry RPC after error@739257152) -> (continuation of Deferred@1372625260 after retry RPC after error@825628392) -> (continuation of Deferred@1573590883 after retry RPC after error@1890352561) -> (continuation of Deferred@635605297 after retry RPC after error@610419251) -> (continuation of Deferred@555170259 after retry RPC after error@843418037) -> (continuation of Deferred@52428245 after retry RPC after error@895112287) -> (continuation of Deferred@756755078 after retry RPC after error@856495793) -> (continuation of Deferred@1325442128 after retry RPC after error@673184067)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.10.81.125:38443
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
01:41:05.422 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
01:41:05.422 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=23, TimeoutTracker(timeout=30000, elapsed=27706), Traces: [0ms] refreshing cache from master, [26ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.126:37005, [88ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.125:38443, [91ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.124:43975, [147ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.125:38443: Network error: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [153ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.126:37005: Network error: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005, [469ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.124:43975: OK, [473ms] delaying RPC due to: Service unavailable: Master config (127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005, [478ms] refreshing cache from master, [479ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.126:37005, [480ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.125:38443, [480ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.124:43975, [485ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.126:37005: Network error: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005, [487ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.125:38443: Network error: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [488ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.124:43975: OK, [489ms] delaying RPC due to: Service unavailable: Master config (127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [498ms] refreshing cache from master, [498ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.126:37005, [499ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.125:38443, [500ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.124:43975, [500ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.126:37005: Network error: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005, [502ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.125:38443: Network error: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [504ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.124:43975: OK, [504ms] delaying RPC due to: Service unavailable: Master config (127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [517ms] refreshing cache from master, [517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.126:37005, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.125:38443, [519ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.124:43975, [519ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.126:37005: Network error: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005, [521ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.125:38443: Network error: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [523ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.124:43975: OK, [524ms] delaying RPC due to: Service unavailable: Master config (127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [537ms] refreshing cache from master, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.126:37005, [538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.125:38443, [538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.124:43975, [539ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.126:37005: Network error: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005, [544ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.125:38443: Network error: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [546ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.124:43975: OK, [548ms] delaying RPC due to: Service unavailable: Master config (127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [577ms] refreshing cache from master, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.126:37005, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.125:38443, [580ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.124:43975, [580ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.126:37005: Network error: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005, [581ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.125:38443: Network error: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [585ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.124:43975: OK, [586ms] delaying RPC due to: Service unavailable: Master config (127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [638ms] refreshing cache from master, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.126:37005, [640ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.125:38443, [641ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.124:43975, [642ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.126:37005: Network error: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005, [644ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.125:38443: Network error: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [645ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.124:43975: OK, [646ms] delaying RPC due to: Service unavailable: Master config (127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [758ms] refreshing cache from master, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.126:37005, [760ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.125:38443, [761ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.124:43975, [761ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.126:37005: Network error: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005, [764ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.125:38443: Network error: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [765ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.124:43975: OK, [766ms] delaying RPC due to: Service unavailable: Master config (127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [917ms] refreshing cache from master, [917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.126:37005, [918ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.125:38443, [919ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.124:43975, [920ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.126:37005: Network error: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005, [924ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.125:38443: Network error: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [925ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.124:43975: OK, [926ms] delaying RPC due to: Service unavailable: Master config (127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [1377ms] refreshing cache from master, [1377ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.126:37005, [1380ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.126:37005: Network error: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005, [1380ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.125:38443, [1381ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.124:43975, [1384ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.125:38443: Network error: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [1387ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.124:43975: OK, [1388ms] delaying RPC due to: Service unavailable: Master config (127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [1417ms] refreshing cache from master, [1417ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.126:37005, [1419ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.126:37005: Network error: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005, [1419ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.125:38443, [1420ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.124:43975, [1421ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.125:38443: Network error: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [1423ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.124:43975: OK, [1424ms] delaying RPC due to: Service unavailable: Master config (127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [2977ms] refreshing cache from master, [2978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.126:37005, [2979ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.125:38443, [2981ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.124:43975, [2981ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.126:37005: Network error: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005, [2983ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.125:38443: Network error: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [2984ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.124:43975: OK, [2985ms] delaying RPC due to: Service unavailable: Master config (127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [6138ms] refreshing cache from master, [6138ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.126:37005, [6139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.125:38443, [6140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.124:43975, [6140ms] too many traces: truncated at 100 traces, deferred=Deferred@505536193(state=PENDING, result=null, callback=(continuation of Deferred@1405071283 after retry RPC after error@665788173) -> (continuation of Deferred@1282917142 after retry RPC after error@1546701711) -> (continuation of Deferred@1918641549 after retry RPC after error@760482748) -> (continuation of Deferred@1607012979 after retry RPC after error@100652726) -> (continuation of Deferred@370998217 after retry RPC after error@462953309) -> (continuation of Deferred@206952663 after retry RPC after error@2003105165) -> (continuation of Deferred@106400283 after retry RPC after error@1565404031) -> (continuation of Deferred@2139079521 after retry RPC after error@803707294) -> (continuation of Deferred@366659883 after retry RPC after error@962792764) -> (continuation of Deferred@948352442 after retry RPC after error@733933661) -> (continuation of Deferred@1760491408 after retry RPC after error@1229262172) -> (continuation of Deferred@1147357867 after retry RPC after error@411264610) -> (continuation of Deferred@122572949 after retry RPC after error@1711313613) -> (continuation of Deferred@946387636 after retry RPC after error@995851181) -> (continuation of Deferred@1323589291 after retry RPC after error@739257152) -> (continuation of Deferred@1372625260 after retry RPC after error@825628392) -> (continuation of Deferred@1573590883 after retry RPC after error@1890352561) -> (continuation of Deferred@635605297 after retry RPC after error@610419251) -> (continuation of Deferred@555170259 after retry RPC after error@843418037) -> (continuation of Deferred@52428245 after retry RPC after error@895112287) -> (continuation of Deferred@756755078 after retry RPC after error@856495793) -> (continuation of Deferred@1325442128 after retry RPC after error@673184067), errback=(continuation of Deferred@1405071283 after retry RPC after error@665788173) -> (continuation of Deferred@1282917142 after retry RPC after error@1546701711) -> (continuation of Deferred@1918641549 after retry RPC after error@760482748) -> (continuation of Deferred@1607012979 after retry RPC after error@100652726) -> (continuation of Deferred@370998217 after retry RPC after error@462953309) -> (continuation of Deferred@206952663 after retry RPC after error@2003105165) -> (continuation of Deferred@106400283 after retry RPC after error@1565404031) -> (continuation of Deferred@2139079521 after retry RPC after error@803707294) -> (continuation of Deferred@366659883 after retry RPC after error@962792764) -> (continuation of Deferred@948352442 after retry RPC after error@733933661) -> (continuation of Deferred@1760491408 after retry RPC after error@1229262172) -> (continuation of Deferred@1147357867 after retry RPC after error@411264610) -> (continuation of Deferred@122572949 after retry RPC after error@1711313613) -> (continuation of Deferred@946387636 after retry RPC after error@995851181) -> (continuation of Deferred@1323589291 after retry RPC after error@739257152) -> (continuation of Deferred@1372625260 after retry RPC after error@825628392) -> (continuation of Deferred@1573590883 after retry RPC after error@1890352561) -> (continuation of Deferred@635605297 after retry RPC after error@610419251) -> (continuation of Deferred@555170259 after retry RPC after error@843418037) -> (continuation of Deferred@52428245 after retry RPC after error@895112287) -> (continuation of Deferred@756755078 after retry RPC after error@856495793) -> (continuation of Deferred@1325442128 after retry RPC after error@673184067)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.10.81.125:38443
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=23, TimeoutTracker(timeout=30000, elapsed=27706), Traces: [0ms] refreshing cache from master, [26ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.126:37005, [88ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.125:38443, [91ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.124:43975, [147ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.125:38443: Network error: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [153ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.126:37005: Network error: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005, [469ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.124:43975: OK, [473ms] delaying RPC due to: Service unavailable: Master config (127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005, [478ms] refreshing cache from master, [479ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.126:37005, [480ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.125:38443, [480ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.124:43975, [485ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.126:37005: Network error: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005, [487ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.125:38443: Network error: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [488ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.124:43975: OK, [489ms] delaying RPC due to: Service unavailable: Master config (127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [498ms] refreshing cache from master, [498ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.126:37005, [499ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.125:38443, [500ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.124:43975, [500ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.126:37005: Network error: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005, [502ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.125:38443: Network error: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [504ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.124:43975: OK, [504ms] delaying RPC due to: Service unavailable: Master config (127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [517ms] refreshing cache from master, [517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.126:37005, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.125:38443, [519ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.124:43975, [519ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.126:37005: Network error: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005, [521ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.125:38443: Network error: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [523ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.124:43975: OK, [524ms] delaying RPC due to: Service unavailable: Master config (127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [537ms] refreshing cache from master, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.126:37005, [538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.125:38443, [538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.124:43975, [539ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.126:37005: Network error: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005, [544ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.125:38443: Network error: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [546ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.124:43975: OK, [548ms] delaying RPC due to: Service unavailable: Master config (127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [577ms] refreshing cache from master, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.126:37005, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.125:38443, [580ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.124:43975, [580ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.126:37005: Network error: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005, [581ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.125:38443: Network error: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [585ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.124:43975: OK, [586ms] delaying RPC due to: Service unavailable: Master config (127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [638ms] refreshing cache from master, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.126:37005, [640ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.125:38443, [641ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.124:43975, [642ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.126:37005: Network error: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005, [644ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.125:38443: Network error: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [645ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.124:43975: OK, [646ms] delaying RPC due to: Service unavailable: Master config (127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [758ms] refreshing cache from master, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.126:37005, [760ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.125:38443, [761ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.124:43975, [761ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.126:37005: Network error: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005, [764ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.125:38443: Network error: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [765ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.124:43975: OK, [766ms] delaying RPC due to: Service unavailable: Master config (127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [917ms] refreshing cache from master, [917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.126:37005, [918ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.125:38443, [919ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.124:43975, [920ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.126:37005: Network error: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005, [924ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.125:38443: Network error: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [925ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.124:43975: OK, [926ms] delaying RPC due to: Service unavailable: Master config (127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [1377ms] refreshing cache from master, [1377ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.126:37005, [1380ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.126:37005: Network error: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005, [1380ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.125:38443, [1381ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.124:43975, [1384ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.125:38443: Network error: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [1387ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.124:43975: OK, [1388ms] delaying RPC due to: Service unavailable: Master config (127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [1417ms] refreshing cache from master, [1417ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.126:37005, [1419ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.126:37005: Network error: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005, [1419ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.125:38443, [1420ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.124:43975, [1421ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.125:38443: Network error: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [1423ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.124:43975: OK, [1424ms] delaying RPC due to: Service unavailable: Master config (127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [2977ms] refreshing cache from master, [2978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.126:37005, [2979ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.125:38443, [2981ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.124:43975, [2981ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.126:37005: Network error: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005, [2983ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.125:38443: Network error: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [2984ms] Sub RPC ConnectToMaster: received response from server master-127.10.81.124:43975: OK, [2985ms] delaying RPC due to: Service unavailable: Master config (127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443, [6138ms] refreshing cache from master, [6138ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.126:37005, [6139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.125:38443, [6140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.81.124:43975, [6140ms] too many traces: truncated at 100 traces, deferred=Deferred@505536193(state=PENDING, result=null, callback=(continuation of Deferred@1405071283 after retry RPC after error@665788173) -> (continuation of Deferred@1282917142 after retry RPC after error@1546701711) -> (continuation of Deferred@1918641549 after retry RPC after error@760482748) -> (continuation of Deferred@1607012979 after retry RPC after error@100652726) -> (continuation of Deferred@370998217 after retry RPC after error@462953309) -> (continuation of Deferred@206952663 after retry RPC after error@2003105165) -> (continuation of Deferred@106400283 after retry RPC after error@1565404031) -> (continuation of Deferred@2139079521 after retry RPC after error@803707294) -> (continuation of Deferred@366659883 after retry RPC after error@962792764) -> (continuation of Deferred@948352442 after retry RPC after error@733933661) -> (continuation of Deferred@1760491408 after retry RPC after error@1229262172) -> (continuation of Deferred@1147357867 after retry RPC after error@411264610) -> (continuation of Deferred@122572949 after retry RPC after error@1711313613) -> (continuation of Deferred@946387636 after retry RPC after error@995851181) -> (continuation of Deferred@1323589291 after retry RPC after error@739257152) -> (continuation of Deferred@1372625260 after retry RPC after error@825628392) -> (continuation of Deferred@1573590883 after retry RPC after error@1890352561) -> (continuation of Deferred@635605297 after retry RPC after error@610419251) -> (continuation of Deferred@555170259 after retry RPC after error@843418037) -> (continuation of Deferred@52428245 after retry RPC after error@895112287) -> (continuation of Deferred@756755078 after retry RPC after error@856495793) -> (continuation of Deferred@1325442128 after retry RPC after error@673184067), errback=(continuation of Deferred@1405071283 after retry RPC after error@665788173) -> (continuation of Deferred@1282917142 after retry RPC after error@1546701711) -> (continuation of Deferred@1918641549 after retry RPC after error@760482748) -> (continuation of Deferred@1607012979 after retry RPC after error@100652726) -> (continuation of Deferred@370998217 after retry RPC after error@462953309) -> (continuation of Deferred@206952663 after retry RPC after error@2003105165) -> (continuation of Deferred@106400283 after retry RPC after error@1565404031) -> (continuation of Deferred@2139079521 after retry RPC after error@803707294) -> (continuation of Deferred@366659883 after retry RPC after error@962792764) -> (continuation of Deferred@948352442 after retry RPC after error@733933661) -> (continuation of Deferred@1760491408 after retry RPC after error@1229262172) -> (continuation of Deferred@1147357867 after retry RPC after error@411264610) -> (continuation of Deferred@122572949 after retry RPC after error@1711313613) -> (continuation of Deferred@946387636 after retry RPC after error@995851181) -> (continuation of Deferred@1323589291 after retry RPC after error@739257152) -> (continuation of Deferred@1372625260 after retry RPC after error@825628392) -> (continuation of Deferred@1573590883 after retry RPC after error@1890352561) -> (continuation of Deferred@635605297 after retry RPC after error@610419251) -> (continuation of Deferred@555170259 after retry RPC after error@843418037) -> (continuation of Deferred@52428245 after retry RPC after error@895112287) -> (continuation of Deferred@756755078 after retry RPC after error@856495793) -> (continuation of Deferred@1325442128 after retry RPC after error@673184067)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.10.81.126:37005,127.10.81.125:38443,127.10.81.124:43975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.126:37005(127.10.81.126:37005): Connection refused: /127.10.81.126:37005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.81.125:38443(127.10.81.125:38443): Connection refused: /127.10.81.125:38443
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.10.81.125:38443
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)