04:01:01.758 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
04:01:01.771 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskKPlGYQ/test-tmp
04:01:01.774 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
04:01:01.777 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskKPlGYQ/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
04:01:02.024 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
04:01:02.174 [DEBUG - main] (MiniKuduCluster.java:195) Response:
04:01:02.176 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
04:01:02.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:02.179898 7911 mini_oidc.cc:152] Starting JWKS server
04:01:02.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:02.183848 7911 webserver.cc:460] Webserver started at http://127.0.0.1:36861/ using document root <none> and password file <none>
04:01:02.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:02.184491 7911 mini_oidc.cc:189] Starting OIDC Discovery server
04:01:02.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:02.186685 7911 webserver.cc:460] Webserver started at http://127.0.0.1:46169/ using document root <none> and password file <none>
04:01:02.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-06T04:01:02Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
04:01:02.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-06T04:01:02Z Disabled control of system clock
04:01:02.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:02.246591 7911 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskKPlGYQ/build/tsan/bin/kudu
04:01:02.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskKPlGYQ/build/tsan/bin/kudu
04:01:02.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/master-0/wal
04:01:02.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/master-0/data
04:01:02.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
04:01:02.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
04:01:02.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
04:01:02.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:01:02.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:01:02.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:01:02.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/master-0/logs
04:01:02.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/master-0/data/info.pb
04:01:02.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:01:02.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
04:01:02.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
04:01:02.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
04:01:02.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
04:01:02.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:01:02.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:01:02.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:01:02.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
04:01:02.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
04:01:02.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:01:02.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:01:02.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.185.254:35943
04:01:02.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.185.254
04:01:02.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:01:02.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.185.212:38181
04:01:02.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:01:02.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:01:02.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:01:02.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:01:02.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.185.254:35943,127.7.185.253:42165,127.7.185.252:37771
04:01:02.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:01:02.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36861/jwks.json with env {}
04:01:02.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:02.560909 7923 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
04:01:02.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:02.561501 7923 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
04:01:02.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:02.562211 7923 flags.cc:425] Enabled unsafe flag: --never_fsync=true
04:01:02.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:02.594169 7923 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
04:01:02.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:02.594475 7923 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
04:01:02.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:02.594666 7923 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
04:01:02.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:02.594933 7923 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
04:01:02.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:02.595199 7923 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
04:01:02.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:02.595439 7923 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36861/jwks.json
04:01:02.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:02.631914 7923 master_runner.cc:386] Master server non-default flags:
04:01:02.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:01:02.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.185.212:38181
04:01:02.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:01:02.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:01:02.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/master-0/data
04:01:02.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/master-0/wal
04:01:02.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:01:02.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.185.254:35943,127.7.185.253:42165,127.7.185.252:37771
04:01:02.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:01:02.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:01:02.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:01:02.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.185.254:35943
04:01:02.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:01:02.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
04:01:02.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:01:02.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36861/jwks.json
04:01:02.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:01:02.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:01:02.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/master-0/data/info.pb
04:01:02.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.185.254
04:01:02.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:01:02.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
04:01:02.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:01:02.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
04:01:02.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
04:01:02.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:01:02.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/master-0/logs
04:01:02.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:01:02.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
04:01:02.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
04:01:02.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
04:01:02.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
04:01:02.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
04:01:02.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
04:01:02.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 02:58:30 UTC on ccycloud.quasar-xnspdp.root.comops.site
04:01:02.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4569537
04:01:02.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
04:01:02.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:02.633535 7923 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
04:01:02.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:02.635823 7923 file_cache.cc:493] Constructed file cache file cache with capacity 419430
04:01:02.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:02.656064 7930 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:01:03.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:02.656093 7929 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:01:03.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:02.657060 7932 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:01:03.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:02.658643 7923 server_base.cc:1034] running on GCE node
04:01:03.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:03.870469 7923 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
04:01:03.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:03.873919 7923 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
04:01:03.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:03.875414 7923 hybrid_clock.cc:648] HybridClock initialized: now 1785988863875401 us; error 109 us; skew 500 ppm
04:01:03.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:03.876395 7923 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
04:01:03.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:03.887401 7923 webserver.cc:460] Webserver started at http://127.7.185.254:42773/ using document root <none> and password file <none>
04:01:03.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:03.888439 7923 fs_manager.cc:359] Metadata directory not provided
04:01:03.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:03.888635 7923 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
04:01:03.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:03.889060 7923 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
04:01:03.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:03.894299 7923 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/master-0/data/instance:
04:01:03.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dc3f36fde382476eaa23085817339ef7"
04:01:03.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 04:01:03 on dist-test-slave-1jjb"
04:01:03.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:03.895462 7923 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/master-0/wal/instance:
04:01:03.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dc3f36fde382476eaa23085817339ef7"
04:01:03.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 04:01:03 on dist-test-slave-1jjb"
04:01:03.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:03.903146 7923 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.001s
04:01:03.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:03.908977 7939 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
04:01:03.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:03.910144 7923 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
04:01:03.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:03.910436 7923 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/master-0/data,/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/master-0/wal
04:01:03.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dc3f36fde382476eaa23085817339ef7"
04:01:03.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 04:01:03 on dist-test-slave-1jjb"
04:01:03.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:03.910739 7923 fs_report.cc:383] FS layout report
04:01:03.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
04:01:03.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/master-0/wal
04:01:03.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/master-0/wal
04:01:03.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/master-0/data/data
04:01:03.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
04:01:03.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
04:01:03.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
04:01:03.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
04:01:03.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
04:01:03.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
04:01:03.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
04:01:03.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
04:01:03.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
04:01:03.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
04:01:03.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
04:01:03.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:03.949637 7923 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
04:01:03.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:03.951154 7923 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
04:01:03.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:03.951575 7923 kserver.cc:163] Server-wide thread pool size limit: 3276
04:01:04.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:04.023517 7923 rpc_server.cc:307] RPC server started. Bound to: 127.7.185.254:35943
04:01:04.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:04.023624 7991 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.185.254:35943 every 8 connection(s)
04:01:04.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:04.026649 7923 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/master-0/data/info.pb
04:01:04.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:04.030658 7911 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskKPlGYQ/build/tsan/bin/kudu as pid 7923
04:01:04.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:04.031234 7911 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/master-0/wal/instance
04:01:04.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:04.032877 7992 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
04:01:04.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:04.036564 7911 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskKPlGYQ/build/tsan/bin/kudu
04:01:04.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskKPlGYQ/build/tsan/bin/kudu
04:01:04.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/master-1/wal
04:01:04.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/master-1/data
04:01:04.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
04:01:04.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
04:01:04.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
04:01:04.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:01:04.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:01:04.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:01:04.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/master-1/logs
04:01:04.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/master-1/data/info.pb
04:01:04.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:01:04.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
04:01:04.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
04:01:04.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
04:01:04.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
04:01:04.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:01:04.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:01:04.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:01:04.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
04:01:04.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
04:01:04.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:01:04.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:01:04.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.185.253:42165
04:01:04.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.185.253
04:01:04.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:01:04.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.185.212:38181
04:01:04.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:01:04.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:01:04.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:01:04.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:01:04.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.185.254:35943,127.7.185.253:42165,127.7.185.252:37771
04:01:04.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:01:04.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36861/jwks.json with env {}
04:01:04.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:04.048197 7992 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.185.254" port: 35943 } has no permanent_uuid. Determining permanent_uuid...
04:01:04.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:04.081465 7992 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.185.253" port: 42165 } has no permanent_uuid. Determining permanent_uuid...
04:01:04.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:04.084122 7940 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.185.253:42165: connect: Connection refused (error 111)
04:01:04.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:04.088236 7992 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.185.253:42165: Network error: Client connection negotiation failed: client connection to 127.7.185.253:42165: connect: Connection refused (error 111)
04:01:04.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:04.137864 7992 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.185.253" port: 42165 } attempt: 1
04:01:04.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:04.141811 7992 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.185.253:42165: Network error: Client connection negotiation failed: client connection to 127.7.185.253:42165: connect: Connection refused (error 111)
04:01:04.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:04.210316 7992 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.185.253" port: 42165 } attempt: 2
04:01:04.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:04.214295 7992 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.185.253:42165: Network error: Client connection negotiation failed: client connection to 127.7.185.253:42165: connect: Connection refused (error 111)
04:01:04.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:04.305791 7992 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.185.253" port: 42165 } attempt: 3
04:01:04.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:04.310308 7992 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.185.253:42165: Network error: Client connection negotiation failed: client connection to 127.7.185.253:42165: connect: Connection refused (error 111)
04:01:04.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:04.359935 7994 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
04:01:04.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:04.360749 7994 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
04:01:04.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:04.361480 7994 flags.cc:425] Enabled unsafe flag: --never_fsync=true
04:01:04.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:04.393296 7994 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
04:01:04.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:04.393698 7994 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
04:01:04.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:04.394023 7994 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
04:01:04.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:04.394383 7994 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
04:01:04.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:04.394733 7994 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
04:01:04.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:04.395012 7994 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36861/jwks.json
04:01:04.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:04.432992 7994 master_runner.cc:386] Master server non-default flags:
04:01:04.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:01:04.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.185.212:38181
04:01:04.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:01:04.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:01:04.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/master-1/data
04:01:04.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/master-1/wal
04:01:04.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:01:04.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.185.254:35943,127.7.185.253:42165,127.7.185.252:37771
04:01:04.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:01:04.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:01:04.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:01:04.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.185.253:42165
04:01:04.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:01:04.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
04:01:04.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:01:04.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36861/jwks.json
04:01:04.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:01:04.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:01:04.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/master-1/data/info.pb
04:01:04.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.185.253
04:01:04.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:01:04.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
04:01:04.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:01:04.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
04:01:04.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
04:01:04.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:01:04.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/master-1/logs
04:01:04.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:01:04.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
04:01:04.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
04:01:04.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
04:01:04.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
04:01:04.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
04:01:04.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
04:01:04.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 02:58:30 UTC on ccycloud.quasar-xnspdp.root.comops.site
04:01:04.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4569537
04:01:04.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
04:01:04.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:04.434633 7994 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
04:01:04.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:04.437042 7994 file_cache.cc:493] Constructed file cache file cache with capacity 419430
04:01:04.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:04.453828 7992 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.185.253" port: 42165 } attempt: 4
04:01:04.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:04.456482 8006 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:01:04.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:04.460487 7992 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.185.253:42165: Network error: Client connection negotiation failed: client connection to 127.7.185.253:42165: connect: Connection refused (error 111)
04:01:04.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:04.760056 7992 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.185.253" port: 42165 } attempt: 5
04:01:04.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:04.766759 7992 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.185.253:42165: Network error: Client connection negotiation failed: client connection to 127.7.185.253:42165: connect: Connection refused (error 111)
04:01:05.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:05.314364 7992 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.185.253" port: 42165 } attempt: 6
04:01:05.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:05.328094 7992 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.185.253:42165: Network error: Client connection negotiation failed: client connection to 127.7.185.253:42165: connect: Connection refused (error 111)
04:01:05.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:04.456758 8004 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:01:05.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:04.457048 8003 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:01:05.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:05.690132 8005 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1230 milliseconds
04:01:05.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:05.690268 7994 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
04:01:05.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:05.691916 7994 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
04:01:05.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:05.695712 7994 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
04:01:05.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:05.697171 7994 hybrid_clock.cc:648] HybridClock initialized: now 1785988865697120 us; error 70 us; skew 500 ppm
04:01:05.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:05.698069 7994 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
04:01:05.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:05.705144 7994 webserver.cc:460] Webserver started at http://127.7.185.253:35227/ using document root <none> and password file <none>
04:01:05.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:05.706100 7994 fs_manager.cc:359] Metadata directory not provided
04:01:05.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:05.706288 7994 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
04:01:05.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:05.706710 7994 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
04:01:05.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:05.711663 7994 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/master-1/data/instance:
04:01:05.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a3ebf087c4964f76bbfe42be9dc46d20"
04:01:05.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 04:01:05 on dist-test-slave-1jjb"
04:01:05.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:05.713064 7994 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/master-1/wal/instance:
04:01:05.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a3ebf087c4964f76bbfe42be9dc46d20"
04:01:05.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 04:01:05 on dist-test-slave-1jjb"
04:01:05.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:05.720827 7994 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
04:01:05.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:05.726631 8015 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
04:01:05.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:05.727836 7994 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.001s
04:01:05.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:05.728196 7994 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/master-1/data,/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/master-1/wal
04:01:05.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a3ebf087c4964f76bbfe42be9dc46d20"
04:01:05.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 04:01:05 on dist-test-slave-1jjb"
04:01:05.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:05.728535 7994 fs_report.cc:383] FS layout report
04:01:05.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
04:01:05.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/master-1/wal
04:01:05.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/master-1/wal
04:01:05.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/master-1/data/data
04:01:05.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
04:01:05.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
04:01:05.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
04:01:05.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
04:01:05.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
04:01:05.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
04:01:05.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
04:01:05.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
04:01:05.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
04:01:05.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
04:01:05.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
04:01:05.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:05.772033 7994 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
04:01:05.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:05.773710 7994 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
04:01:05.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:05.774149 7994 kserver.cc:163] Server-wide thread pool size limit: 3276
04:01:05.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:05.846462 7994 rpc_server.cc:307] RPC server started. Bound to: 127.7.185.253:42165
04:01:05.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:05.846567 8066 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.185.253:42165 every 8 connection(s)
04:01:05.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:05.849413 7994 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/master-1/data/info.pb
04:01:05.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:05.854665 8067 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
04:01:05.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:05.859377 7911 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskKPlGYQ/build/tsan/bin/kudu as pid 7994
04:01:05.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:05.859719 7911 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/master-1/wal/instance
04:01:05.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:05.861807 7911 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskKPlGYQ/build/tsan/bin/kudu
04:01:05.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskKPlGYQ/build/tsan/bin/kudu
04:01:05.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/master-2/wal
04:01:05.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/master-2/data
04:01:05.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
04:01:05.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
04:01:05.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
04:01:05.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:01:05.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:01:05.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:01:05.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/master-2/logs
04:01:05.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/master-2/data/info.pb
04:01:05.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:01:05.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
04:01:05.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
04:01:05.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
04:01:05.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
04:01:05.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:01:05.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:01:05.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:01:05.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
04:01:05.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
04:01:05.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:01:05.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:01:05.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.185.252:37771
04:01:05.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.185.252
04:01:05.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:01:05.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.185.212:38181
04:01:05.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:01:05.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:01:05.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:01:05.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:01:05.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.185.254:35943,127.7.185.253:42165,127.7.185.252:37771
04:01:05.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:01:05.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36861/jwks.json with env {}
04:01:05.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:05.867767 8067 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.185.254" port: 35943 } has no permanent_uuid. Determining permanent_uuid...
04:01:05.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:05.893512 8067 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.185.253" port: 42165 } has no permanent_uuid. Determining permanent_uuid...
04:01:05.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:05.910744 8067 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.185.252" port: 37771 } has no permanent_uuid. Determining permanent_uuid...
04:01:05.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:05.913928 8017 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.185.252:37771: connect: Connection refused (error 111)
04:01:05.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:05.918473 8067 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.185.252:37771: Network error: Client connection negotiation failed: client connection to 127.7.185.252:37771: connect: Connection refused (error 111)
04:01:05.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:05.968149 8067 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.185.252" port: 37771 } attempt: 1
04:01:05.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:05.971889 8067 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.185.252:37771: Network error: Client connection negotiation failed: client connection to 127.7.185.252:37771: connect: Connection refused (error 111)
04:01:06.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:06.040302 8067 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.185.252" port: 37771 } attempt: 2
04:01:06.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:06.044027 8067 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.185.252:37771: Network error: Client connection negotiation failed: client connection to 127.7.185.252:37771: connect: Connection refused (error 111)
04:01:06.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:06.135510 8067 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.185.252" port: 37771 } attempt: 3
04:01:06.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:06.139442 8067 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.185.252:37771: Network error: Client connection negotiation failed: client connection to 127.7.185.252:37771: connect: Connection refused (error 111)
04:01:06.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:06.181430 8069 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
04:01:06.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:06.182003 8069 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
04:01:06.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:06.182463 8069 flags.cc:425] Enabled unsafe flag: --never_fsync=true
04:01:06.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:06.213079 8069 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
04:01:06.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:06.213366 8069 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
04:01:06.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:06.213567 8069 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
04:01:06.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:06.213886 8069 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
04:01:06.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:06.214089 8069 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
04:01:06.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:06.214231 8069 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36861/jwks.json
04:01:06.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:06.250008 8069 master_runner.cc:386] Master server non-default flags:
04:01:06.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:01:06.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.185.212:38181
04:01:06.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:01:06.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:01:06.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/master-2/data
04:01:06.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/master-2/wal
04:01:06.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:01:06.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.185.254:35943,127.7.185.253:42165,127.7.185.252:37771
04:01:06.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:01:06.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:01:06.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:01:06.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.185.252:37771
04:01:06.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:01:06.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
04:01:06.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:01:06.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36861/jwks.json
04:01:06.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:01:06.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:01:06.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/master-2/data/info.pb
04:01:06.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.185.252
04:01:06.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:01:06.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
04:01:06.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:01:06.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
04:01:06.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
04:01:06.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:01:06.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/master-2/logs
04:01:06.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:01:06.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
04:01:06.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
04:01:06.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
04:01:06.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
04:01:06.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
04:01:06.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
04:01:06.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 02:58:30 UTC on ccycloud.quasar-xnspdp.root.comops.site
04:01:06.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4569537
04:01:06.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
04:01:06.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:06.251297 8069 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
04:01:06.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:06.253623 8069 file_cache.cc:493] Constructed file cache file cache with capacity 419430
04:01:06.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:06.275933 8081 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:01:06.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:06.282886 8067 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.185.252" port: 37771 } attempt: 4
04:01:06.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:06.286761 8067 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.185.252:37771: Network error: Client connection negotiation failed: client connection to 127.7.185.252:37771: connect: Connection refused (error 111)
04:01:06.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:06.388855 7992 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.185.253" port: 42165 } attempt: 7
04:01:06.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:06.404765 7992 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.185.252" port: 37771 } has no permanent_uuid. Determining permanent_uuid...
04:01:06.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:06.410200 7992 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.185.252:37771: Network error: Client connection negotiation failed: client connection to 127.7.185.252:37771: connect: Connection refused (error 111)
04:01:06.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:06.468899 7992 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.185.252" port: 37771 } attempt: 1
04:01:06.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:06.474457 7992 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.185.252:37771: Network error: Client connection negotiation failed: client connection to 127.7.185.252:37771: connect: Connection refused (error 111)
04:01:06.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:06.555974 7992 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.185.252" port: 37771 } attempt: 2
04:01:06.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:06.560829 7992 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.185.252:37771: Network error: Client connection negotiation failed: client connection to 127.7.185.252:37771: connect: Connection refused (error 111)
04:01:06.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:06.586174 8067 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.185.252" port: 37771 } attempt: 5
04:01:06.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:06.589891 8067 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.185.252:37771: Network error: Client connection negotiation failed: client connection to 127.7.185.252:37771: connect: Connection refused (error 111)
04:01:06.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:06.646350 7992 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.185.252" port: 37771 } attempt: 3
04:01:06.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:06.651161 7992 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.185.252:37771: Network error: Client connection negotiation failed: client connection to 127.7.185.252:37771: connect: Connection refused (error 111)
04:01:06.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:06.791582 7992 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.185.252" port: 37771 } attempt: 4
04:01:06.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:06.796253 7992 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.185.252:37771: Network error: Client connection negotiation failed: client connection to 127.7.185.252:37771: connect: Connection refused (error 111)
04:01:07.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:07.079888 7992 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.185.252" port: 37771 } attempt: 5
04:01:07.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:07.084190 7992 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.185.252:37771: Network error: Client connection negotiation failed: client connection to 127.7.185.252:37771: connect: Connection refused (error 111)
04:01:07.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:07.137564 8067 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.185.252" port: 37771 } attempt: 6
04:01:07.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:07.144791 8067 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.185.252:37771: Network error: Client connection negotiation failed: client connection to 127.7.185.252:37771: connect: Connection refused (error 111)
04:01:07.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:06.276012 8080 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:01:07.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:06.278241 8069 server_base.cc:1034] running on GCE node
04:01:07.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:06.276103 8083 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:01:07.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:07.492316 8069 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
04:01:07.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:07.494959 8069 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
04:01:07.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:07.496321 8069 hybrid_clock.cc:648] HybridClock initialized: now 1785988867496269 us; error 69 us; skew 500 ppm
04:01:07.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:07.497140 8069 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
04:01:07.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:07.504518 8069 webserver.cc:460] Webserver started at http://127.7.185.252:46405/ using document root <none> and password file <none>
04:01:07.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:07.505478 8069 fs_manager.cc:359] Metadata directory not provided
04:01:07.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:07.505669 8069 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
04:01:07.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:07.506157 8069 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
04:01:07.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:07.511339 8069 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/master-2/data/instance:
04:01:07.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "11616489a4f640cb990ae6e83c914d3c"
04:01:07.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 04:01:07 on dist-test-slave-1jjb"
04:01:07.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:07.512581 8069 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/master-2/wal/instance:
04:01:07.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "11616489a4f640cb990ae6e83c914d3c"
04:01:07.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 04:01:07 on dist-test-slave-1jjb"
04:01:07.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:07.520601 8069 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
04:01:07.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:07.526819 8094 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
04:01:07.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:07.528172 8069 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
04:01:07.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:07.528472 8069 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/master-2/data,/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/master-2/wal
04:01:07.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "11616489a4f640cb990ae6e83c914d3c"
04:01:07.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 04:01:07 on dist-test-slave-1jjb"
04:01:07.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:07.528857 8069 fs_report.cc:383] FS layout report
04:01:07.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
04:01:07.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/master-2/wal
04:01:07.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/master-2/wal
04:01:07.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/master-2/data/data
04:01:07.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
04:01:07.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
04:01:07.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
04:01:07.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
04:01:07.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
04:01:07.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
04:01:07.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
04:01:07.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
04:01:07.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
04:01:07.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
04:01:07.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
04:01:07.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:07.571916 8069 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
04:01:07.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:07.573473 8069 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
04:01:07.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:07.573884 8069 kserver.cc:163] Server-wide thread pool size limit: 3276
04:01:07.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:07.636775 7992 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.185.252" port: 37771 } attempt: 6
04:01:07.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:07.643775 7992 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.185.252:37771: Network error: Client connection negotiation failed: client connection to 127.7.185.252:37771: connect: Connection refused (error 111)
04:01:07.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:07.651568 8069 rpc_server.cc:307] RPC server started. Bound to: 127.7.185.252:37771
04:01:07.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:07.651737 8148 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.185.252:37771 every 8 connection(s)
04:01:07.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:07.654486 8069 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/master-2/data/info.pb
04:01:07.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:07.659924 8149 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
04:01:07.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:07.663010 7911 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskKPlGYQ/build/tsan/bin/kudu as pid 8069
04:01:07.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:07.663492 7911 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/master-2/wal/instance
04:01:07.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:07.678586 8149 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.185.254" port: 35943 } has no permanent_uuid. Determining permanent_uuid...
04:01:07.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:07.707048 8149 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.185.253" port: 42165 } has no permanent_uuid. Determining permanent_uuid...
04:01:07.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:07.720418 8149 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.185.252" port: 37771 } has no permanent_uuid. Determining permanent_uuid...
04:01:07.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:07.737983 8149 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 11616489a4f640cb990ae6e83c914d3c: Bootstrap starting.
04:01:07.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:07.744589 8149 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 11616489a4f640cb990ae6e83c914d3c: Neither blocks nor log segments found. Creating new log.
04:01:07.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:07.746780 8149 log.cc:824] T 00000000000000000000000000000000 P 11616489a4f640cb990ae6e83c914d3c: Log is configured to *not* fsync() on all Append() calls
04:01:07.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:07.753407 8149 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 11616489a4f640cb990ae6e83c914d3c: No bootstrap required, opened a new log
04:01:07.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:07.775627 8149 raft_consensus.cc:348] T 00000000000000000000000000000000 P 11616489a4f640cb990ae6e83c914d3c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc3f36fde382476eaa23085817339ef7" member_type: VOTER last_known_addr { host: "127.7.185.254" port: 35943 } } peers { permanent_uuid: "a3ebf087c4964f76bbfe42be9dc46d20" member_type: VOTER last_known_addr { host: "127.7.185.253" port: 42165 } } peers { permanent_uuid: "11616489a4f640cb990ae6e83c914d3c" member_type: VOTER last_known_addr { host: "127.7.185.252" port: 37771 } }
04:01:07.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:07.777050 8149 raft_consensus.cc:374] T 00000000000000000000000000000000 P 11616489a4f640cb990ae6e83c914d3c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
04:01:07.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:07.777318 8149 raft_consensus.cc:729] T 00000000000000000000000000000000 P 11616489a4f640cb990ae6e83c914d3c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 11616489a4f640cb990ae6e83c914d3c, State: Initialized, Role: FOLLOWER
04:01:07.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:07.778275 8149 consensus_queue.cc:260] T 00000000000000000000000000000000 P 11616489a4f640cb990ae6e83c914d3c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc3f36fde382476eaa23085817339ef7" member_type: VOTER last_known_addr { host: "127.7.185.254" port: 35943 } } peers { permanent_uuid: "a3ebf087c4964f76bbfe42be9dc46d20" member_type: VOTER last_known_addr { host: "127.7.185.253" port: 42165 } } peers { permanent_uuid: "11616489a4f640cb990ae6e83c914d3c" member_type: VOTER last_known_addr { host: "127.7.185.252" port: 37771 } }
04:01:07.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:07.783670 8149 sys_catalog.cc:564] T 00000000000000000000000000000000 P 11616489a4f640cb990ae6e83c914d3c [sys.catalog]: configured and running, proceeding with master startup.
04:01:07.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:07.781579 8158 sys_catalog.cc:455] T 00000000000000000000000000000000 P 11616489a4f640cb990ae6e83c914d3c [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc3f36fde382476eaa23085817339ef7" member_type: VOTER last_known_addr { host: "127.7.185.254" port: 35943 } } peers { permanent_uuid: "a3ebf087c4964f76bbfe42be9dc46d20" member_type: VOTER last_known_addr { host: "127.7.185.253" port: 42165 } } peers { permanent_uuid: "11616489a4f640cb990ae6e83c914d3c" member_type: VOTER last_known_addr { host: "127.7.185.252" port: 37771 } } }
04:01:07.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:07.786728 8158 sys_catalog.cc:458] T 00000000000000000000000000000000 P 11616489a4f640cb990ae6e83c914d3c [sys.catalog]: This master's current role is: FOLLOWER
04:01:07.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:07.811182 8169 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 11616489a4f640cb990ae6e83c914d3c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:01:07.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:07.811959 8169 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:01:07.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:07.829018 8101 tablet.cc:2369] T 00000000000000000000000000000000 P 11616489a4f640cb990ae6e83c914d3c: Can't schedule compaction. Clean time has not been advanced past its initial value.
04:01:07.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:07.807235 8069 master_runner.cc:186] Error getting master registration for 127.7.185.254:35943: OK, instance_id { permanent_uuid: "dc3f36fde382476eaa23085817339ef7" instance_seqno: 1785988864003857 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
04:01:07.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:07.832279 8069 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
04:01:07.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:07.990392 8158 raft_consensus.cc:482] T 00000000000000000000000000000000 P 11616489a4f640cb990ae6e83c914d3c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
04:01:07.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:07.990847 8158 raft_consensus.cc:504] T 00000000000000000000000000000000 P 11616489a4f640cb990ae6e83c914d3c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc3f36fde382476eaa23085817339ef7" member_type: VOTER last_known_addr { host: "127.7.185.254" port: 35943 } } peers { permanent_uuid: "a3ebf087c4964f76bbfe42be9dc46d20" member_type: VOTER last_known_addr { host: "127.7.185.253" port: 42165 } } peers { permanent_uuid: "11616489a4f640cb990ae6e83c914d3c" member_type: VOTER last_known_addr { host: "127.7.185.252" port: 37771 } }
04:01:07.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:07.993234 8158 leader_election.cc:288] T 00000000000000000000000000000000 P 11616489a4f640cb990ae6e83c914d3c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers dc3f36fde382476eaa23085817339ef7 (127.7.185.254:35943), a3ebf087c4964f76bbfe42be9dc46d20 (127.7.185.253:42165)
04:01:08.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.205356 8067 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.185.252" port: 37771 } attempt: 7
04:01:08.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.229139 8067 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a3ebf087c4964f76bbfe42be9dc46d20: Bootstrap starting.
04:01:08.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.236603 8067 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P a3ebf087c4964f76bbfe42be9dc46d20: Neither blocks nor log segments found. Creating new log.
04:01:08.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.238879 8067 log.cc:824] T 00000000000000000000000000000000 P a3ebf087c4964f76bbfe42be9dc46d20: Log is configured to *not* fsync() on all Append() calls
04:01:08.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.243923 8067 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a3ebf087c4964f76bbfe42be9dc46d20: No bootstrap required, opened a new log
04:01:08.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.273065 8067 raft_consensus.cc:348] T 00000000000000000000000000000000 P a3ebf087c4964f76bbfe42be9dc46d20 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc3f36fde382476eaa23085817339ef7" member_type: VOTER last_known_addr { host: "127.7.185.254" port: 35943 } } peers { permanent_uuid: "a3ebf087c4964f76bbfe42be9dc46d20" member_type: VOTER last_known_addr { host: "127.7.185.253" port: 42165 } } peers { permanent_uuid: "11616489a4f640cb990ae6e83c914d3c" member_type: VOTER last_known_addr { host: "127.7.185.252" port: 37771 } }
04:01:08.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.273732 8067 raft_consensus.cc:374] T 00000000000000000000000000000000 P a3ebf087c4964f76bbfe42be9dc46d20 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
04:01:08.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.273983 8067 raft_consensus.cc:729] T 00000000000000000000000000000000 P a3ebf087c4964f76bbfe42be9dc46d20 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a3ebf087c4964f76bbfe42be9dc46d20, State: Initialized, Role: FOLLOWER
04:01:08.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.274912 8067 consensus_queue.cc:260] T 00000000000000000000000000000000 P a3ebf087c4964f76bbfe42be9dc46d20 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc3f36fde382476eaa23085817339ef7" member_type: VOTER last_known_addr { host: "127.7.185.254" port: 35943 } } peers { permanent_uuid: "a3ebf087c4964f76bbfe42be9dc46d20" member_type: VOTER last_known_addr { host: "127.7.185.253" port: 42165 } } peers { permanent_uuid: "11616489a4f640cb990ae6e83c914d3c" member_type: VOTER last_known_addr { host: "127.7.185.252" port: 37771 } }
04:01:08.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.278661 8174 sys_catalog.cc:455] T 00000000000000000000000000000000 P a3ebf087c4964f76bbfe42be9dc46d20 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc3f36fde382476eaa23085817339ef7" member_type: VOTER last_known_addr { host: "127.7.185.254" port: 35943 } } peers { permanent_uuid: "a3ebf087c4964f76bbfe42be9dc46d20" member_type: VOTER last_known_addr { host: "127.7.185.253" port: 42165 } } peers { permanent_uuid: "11616489a4f640cb990ae6e83c914d3c" member_type: VOTER last_known_addr { host: "127.7.185.252" port: 37771 } } }
04:01:08.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.279443 8174 sys_catalog.cc:458] T 00000000000000000000000000000000 P a3ebf087c4964f76bbfe42be9dc46d20 [sys.catalog]: This master's current role is: FOLLOWER
04:01:08.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.279666 8067 sys_catalog.cc:564] T 00000000000000000000000000000000 P a3ebf087c4964f76bbfe42be9dc46d20 [sys.catalog]: configured and running, proceeding with master startup.
04:01:08.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.279852 8042 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "11616489a4f640cb990ae6e83c914d3c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a3ebf087c4964f76bbfe42be9dc46d20" is_pre_election: true
04:01:08.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.281055 8042 raft_consensus.cc:2454] T 00000000000000000000000000000000 P a3ebf087c4964f76bbfe42be9dc46d20 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 11616489a4f640cb990ae6e83c914d3c in term 0.
04:01:08.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:08.282047 8021 tablet.cc:2369] T 00000000000000000000000000000000 P a3ebf087c4964f76bbfe42be9dc46d20: Can't schedule compaction. Clean time has not been advanced past its initial value.
04:01:08.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.282835 8096 leader_election.cc:302] T 00000000000000000000000000000000 P 11616489a4f640cb990ae6e83c914d3c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 11616489a4f640cb990ae6e83c914d3c, a3ebf087c4964f76bbfe42be9dc46d20; no voters:
04:01:08.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.283773 8158 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 11616489a4f640cb990ae6e83c914d3c [term 0 FOLLOWER]: Leader pre-election won for term 1
04:01:08.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.284247 8158 raft_consensus.cc:482] T 00000000000000000000000000000000 P 11616489a4f640cb990ae6e83c914d3c [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
04:01:08.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.284664 8158 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 11616489a4f640cb990ae6e83c914d3c [term 0 FOLLOWER]: Advancing to term 1
04:01:08.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.294350 8158 raft_consensus.cc:504] T 00000000000000000000000000000000 P 11616489a4f640cb990ae6e83c914d3c [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc3f36fde382476eaa23085817339ef7" member_type: VOTER last_known_addr { host: "127.7.185.254" port: 35943 } } peers { permanent_uuid: "a3ebf087c4964f76bbfe42be9dc46d20" member_type: VOTER last_known_addr { host: "127.7.185.253" port: 42165 } } peers { permanent_uuid: "11616489a4f640cb990ae6e83c914d3c" member_type: VOTER last_known_addr { host: "127.7.185.252" port: 37771 } }
04:01:08.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.300012 8158 leader_election.cc:288] T 00000000000000000000000000000000 P 11616489a4f640cb990ae6e83c914d3c [CANDIDATE]: Term 1 election: Requested vote from peers dc3f36fde382476eaa23085817339ef7 (127.7.185.254:35943), a3ebf087c4964f76bbfe42be9dc46d20 (127.7.185.253:42165)
04:01:08.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.301000 8042 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "11616489a4f640cb990ae6e83c914d3c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a3ebf087c4964f76bbfe42be9dc46d20"
04:01:08.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.301612 8042 raft_consensus.cc:3037] T 00000000000000000000000000000000 P a3ebf087c4964f76bbfe42be9dc46d20 [term 0 FOLLOWER]: Advancing to term 1
04:01:08.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.314477 8042 raft_consensus.cc:2454] T 00000000000000000000000000000000 P a3ebf087c4964f76bbfe42be9dc46d20 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 11616489a4f640cb990ae6e83c914d3c in term 1.
04:01:08.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.315959 8096 leader_election.cc:302] T 00000000000000000000000000000000 P 11616489a4f640cb990ae6e83c914d3c [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 11616489a4f640cb990ae6e83c914d3c, a3ebf087c4964f76bbfe42be9dc46d20; no voters:
04:01:08.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.316839 8158 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 11616489a4f640cb990ae6e83c914d3c [term 1 FOLLOWER]: Leader election won for term 1
04:01:08.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.327710 8158 raft_consensus.cc:686] T 00000000000000000000000000000000 P 11616489a4f640cb990ae6e83c914d3c [term 1 LEADER]: Becoming Leader. State: Replica: 11616489a4f640cb990ae6e83c914d3c, State: Running, Role: LEADER
04:01:08.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.328980 8158 consensus_queue.cc:237] T 00000000000000000000000000000000 P 11616489a4f640cb990ae6e83c914d3c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc3f36fde382476eaa23085817339ef7" member_type: VOTER last_known_addr { host: "127.7.185.254" port: 35943 } } peers { permanent_uuid: "a3ebf087c4964f76bbfe42be9dc46d20" member_type: VOTER last_known_addr { host: "127.7.185.253" port: 42165 } } peers { permanent_uuid: "11616489a4f640cb990ae6e83c914d3c" member_type: VOTER last_known_addr { host: "127.7.185.252" port: 37771 } }
04:01:08.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.339641 8182 sys_catalog.cc:455] T 00000000000000000000000000000000 P 11616489a4f640cb990ae6e83c914d3c [sys.catalog]: SysCatalogTable state changed. Reason: New leader 11616489a4f640cb990ae6e83c914d3c. Latest consensus state: current_term: 1 leader_uuid: "11616489a4f640cb990ae6e83c914d3c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc3f36fde382476eaa23085817339ef7" member_type: VOTER last_known_addr { host: "127.7.185.254" port: 35943 } } peers { permanent_uuid: "a3ebf087c4964f76bbfe42be9dc46d20" member_type: VOTER last_known_addr { host: "127.7.185.253" port: 42165 } } peers { permanent_uuid: "11616489a4f640cb990ae6e83c914d3c" member_type: VOTER last_known_addr { host: "127.7.185.252" port: 37771 } } }
04:01:08.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.340775 8182 sys_catalog.cc:458] T 00000000000000000000000000000000 P 11616489a4f640cb990ae6e83c914d3c [sys.catalog]: This master's current role is: LEADER
04:01:08.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.354058 8188 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
04:01:08.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:08.371744 8187 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a3ebf087c4964f76bbfe42be9dc46d20: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:01:08.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:08.372185 8187 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:01:08.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.373926 8188 catalog_manager.cc:1484] Initializing Kudu cluster ID...
04:01:08.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.353996 7994 master_runner.cc:186] Error getting master registration for 127.7.185.254:35943: OK, instance_id { permanent_uuid: "dc3f36fde382476eaa23085817339ef7" instance_seqno: 1785988864003857 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
04:01:08.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.388589 7994 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
04:01:08.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.401377 8042 raft_consensus.cc:1261] T 00000000000000000000000000000000 P a3ebf087c4964f76bbfe42be9dc46d20 [term 1 FOLLOWER]: Refusing update from remote peer 11616489a4f640cb990ae6e83c914d3c: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
04:01:08.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.403324 8158 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 11616489a4f640cb990ae6e83c914d3c [LEADER]: Connected to new peer: Peer: permanent_uuid: "a3ebf087c4964f76bbfe42be9dc46d20" member_type: VOTER last_known_addr { host: "127.7.185.253" port: 42165 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
04:01:08.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.444986 8174 sys_catalog.cc:455] T 00000000000000000000000000000000 P a3ebf087c4964f76bbfe42be9dc46d20 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 11616489a4f640cb990ae6e83c914d3c. Latest consensus state: current_term: 1 leader_uuid: "11616489a4f640cb990ae6e83c914d3c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc3f36fde382476eaa23085817339ef7" member_type: VOTER last_known_addr { host: "127.7.185.254" port: 35943 } } peers { permanent_uuid: "a3ebf087c4964f76bbfe42be9dc46d20" member_type: VOTER last_known_addr { host: "127.7.185.253" port: 42165 } } peers { permanent_uuid: "11616489a4f640cb990ae6e83c914d3c" member_type: VOTER last_known_addr { host: "127.7.185.252" port: 37771 } } }
04:01:08.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.445856 8174 sys_catalog.cc:458] T 00000000000000000000000000000000 P a3ebf087c4964f76bbfe42be9dc46d20 [sys.catalog]: This master's current role is: FOLLOWER
04:01:08.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.460446 8182 sys_catalog.cc:455] T 00000000000000000000000000000000 P 11616489a4f640cb990ae6e83c914d3c [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "11616489a4f640cb990ae6e83c914d3c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc3f36fde382476eaa23085817339ef7" member_type: VOTER last_known_addr { host: "127.7.185.254" port: 35943 } } peers { permanent_uuid: "a3ebf087c4964f76bbfe42be9dc46d20" member_type: VOTER last_known_addr { host: "127.7.185.253" port: 42165 } } peers { permanent_uuid: "11616489a4f640cb990ae6e83c914d3c" member_type: VOTER last_known_addr { host: "127.7.185.252" port: 37771 } } }
04:01:08.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.461272 8182 sys_catalog.cc:458] T 00000000000000000000000000000000 P 11616489a4f640cb990ae6e83c914d3c [sys.catalog]: This master's current role is: LEADER
04:01:08.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.466820 8188 catalog_manager.cc:1347] Generated new cluster ID: 775d72d2385d486082294614a303ad2a
04:01:08.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.467204 8188 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
04:01:08.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.479369 8174 sys_catalog.cc:455] T 00000000000000000000000000000000 P a3ebf087c4964f76bbfe42be9dc46d20 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "11616489a4f640cb990ae6e83c914d3c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc3f36fde382476eaa23085817339ef7" member_type: VOTER last_known_addr { host: "127.7.185.254" port: 35943 } } peers { permanent_uuid: "a3ebf087c4964f76bbfe42be9dc46d20" member_type: VOTER last_known_addr { host: "127.7.185.253" port: 42165 } } peers { permanent_uuid: "11616489a4f640cb990ae6e83c914d3c" member_type: VOTER last_known_addr { host: "127.7.185.252" port: 37771 } } }
04:01:08.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.480111 8174 sys_catalog.cc:458] T 00000000000000000000000000000000 P a3ebf087c4964f76bbfe42be9dc46d20 [sys.catalog]: This master's current role is: FOLLOWER
04:01:08.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.505803 8188 catalog_manager.cc:1370] Generated new certificate authority record
04:01:08.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.512125 8188 catalog_manager.cc:1504] Loading token signing keys...
04:01:08.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.552055 8188 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 11616489a4f640cb990ae6e83c914d3c: Generated new TSK 0
04:01:08.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.555703 8188 catalog_manager.cc:1514] Initializing in-progress tserver states...
04:01:08.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.579619 7911 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskKPlGYQ/build/tsan/bin/kudu
04:01:08.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskKPlGYQ/build/tsan/bin/kudu
04:01:08.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/ts-0/wal
04:01:08.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/ts-0/data
04:01:08.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
04:01:08.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
04:01:08.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
04:01:08.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:01:08.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:01:08.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:01:08.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/ts-0/logs
04:01:08.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/ts-0/data/info.pb
04:01:08.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:01:08.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
04:01:08.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
04:01:08.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
04:01:08.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
04:01:08.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:01:08.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:01:08.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:01:08.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
04:01:08.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
04:01:08.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.185.193:0
04:01:08.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.7.185.193
04:01:08.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.185.193
04:01:08.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:01:08.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.7.185.254:35943,127.7.185.253:42165,127.7.185.252:37771
04:01:08.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.185.212:38181
04:01:08.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:01:08.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:01:08.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
04:01:08.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.677177 7992 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.185.252" port: 37771 } attempt: 7
04:01:08.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.730103 7992 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P dc3f36fde382476eaa23085817339ef7: Bootstrap starting.
04:01:08.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.744545 7992 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P dc3f36fde382476eaa23085817339ef7: Neither blocks nor log segments found. Creating new log.
04:01:08.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.747500 7992 log.cc:824] T 00000000000000000000000000000000 P dc3f36fde382476eaa23085817339ef7: Log is configured to *not* fsync() on all Append() calls
04:01:08.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.783793 7992 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P dc3f36fde382476eaa23085817339ef7: No bootstrap required, opened a new log
04:01:08.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.847914 7992 raft_consensus.cc:348] T 00000000000000000000000000000000 P dc3f36fde382476eaa23085817339ef7 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc3f36fde382476eaa23085817339ef7" member_type: VOTER last_known_addr { host: "127.7.185.254" port: 35943 } } peers { permanent_uuid: "a3ebf087c4964f76bbfe42be9dc46d20" member_type: VOTER last_known_addr { host: "127.7.185.253" port: 42165 } } peers { permanent_uuid: "11616489a4f640cb990ae6e83c914d3c" member_type: VOTER last_known_addr { host: "127.7.185.252" port: 37771 } }
04:01:08.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.849013 7992 raft_consensus.cc:374] T 00000000000000000000000000000000 P dc3f36fde382476eaa23085817339ef7 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
04:01:08.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.849309 7992 raft_consensus.cc:729] T 00000000000000000000000000000000 P dc3f36fde382476eaa23085817339ef7 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: dc3f36fde382476eaa23085817339ef7, State: Initialized, Role: FOLLOWER
04:01:08.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.850318 7992 consensus_queue.cc:260] T 00000000000000000000000000000000 P dc3f36fde382476eaa23085817339ef7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc3f36fde382476eaa23085817339ef7" member_type: VOTER last_known_addr { host: "127.7.185.254" port: 35943 } } peers { permanent_uuid: "a3ebf087c4964f76bbfe42be9dc46d20" member_type: VOTER last_known_addr { host: "127.7.185.253" port: 42165 } } peers { permanent_uuid: "11616489a4f640cb990ae6e83c914d3c" member_type: VOTER last_known_addr { host: "127.7.185.252" port: 37771 } }
04:01:08.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.854662 7992 sys_catalog.cc:564] T 00000000000000000000000000000000 P dc3f36fde382476eaa23085817339ef7 [sys.catalog]: configured and running, proceeding with master startup.
04:01:08.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.856855 7965 raft_consensus.cc:3037] T 00000000000000000000000000000000 P dc3f36fde382476eaa23085817339ef7 [term 0 FOLLOWER]: Advancing to term 1
04:01:08.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.863497 7967 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "11616489a4f640cb990ae6e83c914d3c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "dc3f36fde382476eaa23085817339ef7" is_pre_election: true
04:01:08.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.864323 7966 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "11616489a4f640cb990ae6e83c914d3c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "dc3f36fde382476eaa23085817339ef7"
04:01:08.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.872854 7965 raft_consensus.cc:1261] T 00000000000000000000000000000000 P dc3f36fde382476eaa23085817339ef7 [term 1 FOLLOWER]: Refusing update from remote peer 11616489a4f640cb990ae6e83c914d3c: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
04:01:08.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.876619 8158 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 11616489a4f640cb990ae6e83c914d3c [LEADER]: Connected to new peer: Peer: permanent_uuid: "dc3f36fde382476eaa23085817339ef7" member_type: VOTER last_known_addr { host: "127.7.185.254" port: 35943 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
04:01:08.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.878939 8198 sys_catalog.cc:455] T 00000000000000000000000000000000 P dc3f36fde382476eaa23085817339ef7 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc3f36fde382476eaa23085817339ef7" member_type: VOTER last_known_addr { host: "127.7.185.254" port: 35943 } } peers { permanent_uuid: "a3ebf087c4964f76bbfe42be9dc46d20" member_type: VOTER last_known_addr { host: "127.7.185.253" port: 42165 } } peers { permanent_uuid: "11616489a4f640cb990ae6e83c914d3c" member_type: VOTER last_known_addr { host: "127.7.185.252" port: 37771 } } }
04:01:08.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.879889 8198 sys_catalog.cc:458] T 00000000000000000000000000000000 P dc3f36fde382476eaa23085817339ef7 [sys.catalog]: This master's current role is: FOLLOWER
04:01:08.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.946871 8198 sys_catalog.cc:455] T 00000000000000000000000000000000 P dc3f36fde382476eaa23085817339ef7 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 11616489a4f640cb990ae6e83c914d3c. Latest consensus state: current_term: 1 leader_uuid: "11616489a4f640cb990ae6e83c914d3c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc3f36fde382476eaa23085817339ef7" member_type: VOTER last_known_addr { host: "127.7.185.254" port: 35943 } } peers { permanent_uuid: "a3ebf087c4964f76bbfe42be9dc46d20" member_type: VOTER last_known_addr { host: "127.7.185.253" port: 42165 } } peers { permanent_uuid: "11616489a4f640cb990ae6e83c914d3c" member_type: VOTER last_known_addr { host: "127.7.185.252" port: 37771 } } }
04:01:08.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.947060 8206 sys_catalog.cc:455] T 00000000000000000000000000000000 P dc3f36fde382476eaa23085817339ef7 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "11616489a4f640cb990ae6e83c914d3c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc3f36fde382476eaa23085817339ef7" member_type: VOTER last_known_addr { host: "127.7.185.254" port: 35943 } } peers { permanent_uuid: "a3ebf087c4964f76bbfe42be9dc46d20" member_type: VOTER last_known_addr { host: "127.7.185.253" port: 42165 } } peers { permanent_uuid: "11616489a4f640cb990ae6e83c914d3c" member_type: VOTER last_known_addr { host: "127.7.185.252" port: 37771 } } }
04:01:08.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.947741 8198 sys_catalog.cc:458] T 00000000000000000000000000000000 P dc3f36fde382476eaa23085817339ef7 [sys.catalog]: This master's current role is: FOLLOWER
04:01:08.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.947815 8206 sys_catalog.cc:458] T 00000000000000000000000000000000 P dc3f36fde382476eaa23085817339ef7 [sys.catalog]: This master's current role is: FOLLOWER
04:01:08.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.986539 8214 catalog_manager.cc:1259] Loaded cluster ID: 775d72d2385d486082294614a303ad2a
04:01:08.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.987250 8214 catalog_manager.cc:1552] T 00000000000000000000000000000000 P dc3f36fde382476eaa23085817339ef7: loading cluster ID for follower catalog manager: success
04:01:08.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:08.999419 8214 catalog_manager.cc:1574] T 00000000000000000000000000000000 P dc3f36fde382476eaa23085817339ef7: acquiring CA information for follower catalog manager: success
04:01:09.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:09.018185 8214 catalog_manager.cc:1602] T 00000000000000000000000000000000 P dc3f36fde382476eaa23085817339ef7: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
04:01:09.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:09.312000 8195 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
04:01:09.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:09.313043 8195 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
04:01:09.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:09.314061 8195 flags.cc:425] Enabled unsafe flag: --never_fsync=true
04:01:09.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:09.364941 8221 sys_catalog.cc:455] T 00000000000000000000000000000000 P 11616489a4f640cb990ae6e83c914d3c [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "11616489a4f640cb990ae6e83c914d3c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc3f36fde382476eaa23085817339ef7" member_type: VOTER last_known_addr { host: "127.7.185.254" port: 35943 } } peers { permanent_uuid: "a3ebf087c4964f76bbfe42be9dc46d20" member_type: VOTER last_known_addr { host: "127.7.185.253" port: 42165 } } peers { permanent_uuid: "11616489a4f640cb990ae6e83c914d3c" member_type: VOTER last_known_addr { host: "127.7.185.252" port: 37771 } } }
04:01:09.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:09.366129 8221 sys_catalog.cc:458] T 00000000000000000000000000000000 P 11616489a4f640cb990ae6e83c914d3c [sys.catalog]: This master's current role is: LEADER
04:01:09.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:09.378602 8187 catalog_manager.cc:1259] Loaded cluster ID: 775d72d2385d486082294614a303ad2a
04:01:09.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:09.379205 8187 catalog_manager.cc:1552] T 00000000000000000000000000000000 P a3ebf087c4964f76bbfe42be9dc46d20: loading cluster ID for follower catalog manager: success
04:01:09.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:09.387531 8187 catalog_manager.cc:1574] T 00000000000000000000000000000000 P a3ebf087c4964f76bbfe42be9dc46d20: acquiring CA information for follower catalog manager: success
04:01:09.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:09.397217 8195 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
04:01:09.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:09.398671 8195 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.185.193
04:01:09.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:09.403677 8187 catalog_manager.cc:1602] T 00000000000000000000000000000000 P a3ebf087c4964f76bbfe42be9dc46d20: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
04:01:09.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:09.463721 8195 tablet_server_runner.cc:78] Tablet server non-default flags:
04:01:09.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:01:09.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.185.212:38181
04:01:09.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:01:09.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:01:09.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/ts-0/data
04:01:09.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/ts-0/wal
04:01:09.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:01:09.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:01:09.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.185.193:0
04:01:09.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
04:01:09.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:01:09.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:01:09.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/ts-0/data/info.pb
04:01:09.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.185.193
04:01:09.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:01:09.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.7.185.254:35943,127.7.185.253:42165,127.7.185.252:37771
04:01:09.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
04:01:09.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:01:09.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
04:01:09.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
04:01:09.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:01:09.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.7.185.193
04:01:09.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/ts-0/logs
04:01:09.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:01:09.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
04:01:09.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
04:01:09.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
04:01:09.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
04:01:09.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
04:01:09.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
04:01:09.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 02:58:30 UTC on ccycloud.quasar-xnspdp.root.comops.site
04:01:09.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4569537
04:01:09.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
04:01:09.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:09.465619 8195 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
04:01:09.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:09.467792 8195 file_cache.cc:493] Constructed file cache file cache with capacity 419430
04:01:09.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:09.521183 8225 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:01:11.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:09.525734 8195 server_base.cc:1034] running on GCE node
04:01:11.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:09.530437 8228 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:01:11.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:09.524242 8224 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:01:11.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:11.086699 8195 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
04:01:11.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:11.090585 8195 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
04:01:11.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:11.092192 8195 hybrid_clock.cc:648] HybridClock initialized: now 1785988871092115 us; error 82 us; skew 500 ppm
04:01:11.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:11.093474 8195 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
04:01:11.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:11.108996 8195 webserver.cc:460] Webserver started at http://127.7.185.193:38247/ using document root <none> and password file <none>
04:01:11.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:11.110427 8195 fs_manager.cc:359] Metadata directory not provided
04:01:11.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:11.111009 8195 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
04:01:11.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:11.111662 8195 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
04:01:11.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:11.119590 8195 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/ts-0/data/instance:
04:01:11.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1269a115c7804e08bbc9a1af11429ad4"
04:01:11.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 04:01:11 on dist-test-slave-1jjb"
04:01:11.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:11.121326 8195 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/ts-0/wal/instance:
04:01:11.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1269a115c7804e08bbc9a1af11429ad4"
04:01:11.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 04:01:11 on dist-test-slave-1jjb"
04:01:11.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:11.132275 8195 fs_manager.cc:691] Time spent creating directory manager: real 0.010s user 0.006s sys 0.005s
04:01:11.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:11.140511 8234 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
04:01:11.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:11.141929 8195 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.003s sys 0.001s
04:01:11.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:11.142336 8195 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/ts-0/data,/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/ts-0/wal
04:01:11.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1269a115c7804e08bbc9a1af11429ad4"
04:01:11.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 04:01:11 on dist-test-slave-1jjb"
04:01:11.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:11.142781 8195 fs_report.cc:383] FS layout report
04:01:11.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
04:01:11.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/ts-0/wal
04:01:11.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/ts-0/wal
04:01:11.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/ts-0/data/data
04:01:11.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
04:01:11.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
04:01:11.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
04:01:11.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
04:01:11.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
04:01:11.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
04:01:11.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
04:01:11.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
04:01:11.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
04:01:11.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
04:01:11.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
04:01:11.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:11.191077 8195 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
04:01:11.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:11.193135 8195 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
04:01:11.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:11.193714 8195 kserver.cc:163] Server-wide thread pool size limit: 3276
04:01:11.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:11.197387 8195 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
04:01:11.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:11.203473 8195 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
04:01:11.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:11.203845 8195 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
04:01:11.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:11.204257 8195 ts_tablet_manager.cc:610] Registered 0 tablets
04:01:11.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:11.204490 8195 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
04:01:11.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:11.408638 8195 rpc_server.cc:307] RPC server started. Bound to: 127.7.185.193:38495
04:01:11.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:11.408741 8349 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.185.193:38495 every 8 connection(s)
04:01:11.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:11.411603 8195 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/ts-0/data/info.pb
04:01:11.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:11.414436 7911 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskKPlGYQ/build/tsan/bin/kudu as pid 8195
04:01:11.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:11.415041 7911 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/ts-0/wal/instance
04:01:11.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:11.428917 7911 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskKPlGYQ/build/tsan/bin/kudu
04:01:11.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskKPlGYQ/build/tsan/bin/kudu
04:01:11.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/ts-1/wal
04:01:11.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/ts-1/data
04:01:11.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
04:01:11.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
04:01:11.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
04:01:11.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:01:11.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:01:11.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:01:11.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/ts-1/logs
04:01:11.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/ts-1/data/info.pb
04:01:11.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:01:11.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
04:01:11.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
04:01:11.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
04:01:11.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
04:01:11.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:01:11.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:01:11.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:01:11.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
04:01:11.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
04:01:11.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.185.194:0
04:01:11.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.7.185.194
04:01:11.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.185.194
04:01:11.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:01:11.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.7.185.254:35943,127.7.185.253:42165,127.7.185.252:37771
04:01:11.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.185.212:38181
04:01:11.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:01:11.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:01:11.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
04:01:11.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:11.492589 8356 heartbeater.cc:347] Connected to a master server at 127.7.185.254:35943
04:01:11.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:11.493129 8356 heartbeater.cc:464] Registering TS with master...
04:01:11.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:11.494334 8356 heartbeater.cc:511] Master 127.7.185.254:35943 requested a full tablet report, sending...
04:01:11.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:11.498080 7956 ts_manager.cc:194] Registered new tserver with Master: 1269a115c7804e08bbc9a1af11429ad4 (127.7.185.193:38495)
04:01:11.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:11.509627 8351 heartbeater.cc:347] Connected to a master server at 127.7.185.253:42165
04:01:11.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:11.510082 8351 heartbeater.cc:464] Registering TS with master...
04:01:11.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:11.509649 8350 heartbeater.cc:347] Connected to a master server at 127.7.185.252:37771
04:01:11.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:11.510643 8350 heartbeater.cc:464] Registering TS with master...
04:01:11.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:11.510977 8351 heartbeater.cc:511] Master 127.7.185.253:42165 requested a full tablet report, sending...
04:01:11.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:11.511485 8350 heartbeater.cc:511] Master 127.7.185.252:37771 requested a full tablet report, sending...
04:01:11.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:11.514451 8032 ts_manager.cc:194] Registered new tserver with Master: 1269a115c7804e08bbc9a1af11429ad4 (127.7.185.193:38495)
04:01:11.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:11.517164 8112 ts_manager.cc:194] Registered new tserver with Master: 1269a115c7804e08bbc9a1af11429ad4 (127.7.185.193:38495)
04:01:11.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:11.519315 8112 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.185.193:46365
04:01:11.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:11.864405 8360 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
04:01:11.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:11.864903 8360 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
04:01:11.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:11.865583 8360 flags.cc:425] Enabled unsafe flag: --never_fsync=true
04:01:11.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:11.897612 8360 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
04:01:11.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:11.898420 8360 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.185.194
04:01:11.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:11.934146 8360 tablet_server_runner.cc:78] Tablet server non-default flags:
04:01:11.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:01:11.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.185.212:38181
04:01:11.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:01:11.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:01:11.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/ts-1/data
04:01:11.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/ts-1/wal
04:01:11.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:01:11.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:01:11.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.185.194:0
04:01:11.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
04:01:11.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:01:11.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:01:11.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/ts-1/data/info.pb
04:01:11.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.185.194
04:01:11.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:01:11.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.7.185.254:35943,127.7.185.253:42165,127.7.185.252:37771
04:01:11.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
04:01:11.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:01:11.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
04:01:11.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
04:01:11.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:01:11.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.7.185.194
04:01:11.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/ts-1/logs
04:01:11.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:01:11.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
04:01:11.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
04:01:11.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
04:01:11.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
04:01:11.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
04:01:11.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
04:01:11.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 02:58:30 UTC on ccycloud.quasar-xnspdp.root.comops.site
04:01:11.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4569537
04:01:11.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
04:01:11.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:11.935756 8360 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
04:01:11.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:11.937870 8360 file_cache.cc:493] Constructed file cache file cache with capacity 419430
04:01:11.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:11.957432 8366 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:01:12.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:12.524467 8350 heartbeater.cc:503] Master 127.7.185.252:37771 was elected leader, sending a full tablet report...
04:01:13.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:11.957477 8369 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:01:13.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:11.957516 8367 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:01:13.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:13.194190 8368 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1236 milliseconds
04:01:13.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:13.194316 8360 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
04:01:13.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:13.195533 8360 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
04:01:13.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:13.197829 8360 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
04:01:13.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:13.199198 8360 hybrid_clock.cc:648] HybridClock initialized: now 1785988873199167 us; error 42 us; skew 500 ppm
04:01:13.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:13.199963 8360 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
04:01:13.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:13.206178 8360 webserver.cc:460] Webserver started at http://127.7.185.194:41337/ using document root <none> and password file <none>
04:01:13.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:13.207056 8360 fs_manager.cc:359] Metadata directory not provided
04:01:13.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:13.207238 8360 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
04:01:13.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:13.207623 8360 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
04:01:13.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:13.212715 8360 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/ts-1/data/instance:
04:01:13.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1ad0296afe7c432db3797611d1facdc4"
04:01:13.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 04:01:13 on dist-test-slave-1jjb"
04:01:13.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:13.213778 8360 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/ts-1/wal/instance:
04:01:13.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1ad0296afe7c432db3797611d1facdc4"
04:01:13.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 04:01:13 on dist-test-slave-1jjb"
04:01:13.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:13.220930 8360 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.003s sys 0.005s
04:01:13.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:13.226410 8378 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
04:01:13.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:13.227481 8360 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
04:01:13.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:13.227766 8360 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/ts-1/data,/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/ts-1/wal
04:01:13.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1ad0296afe7c432db3797611d1facdc4"
04:01:13.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 04:01:13 on dist-test-slave-1jjb"
04:01:13.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:13.228070 8360 fs_report.cc:383] FS layout report
04:01:13.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
04:01:13.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/ts-1/wal
04:01:13.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/ts-1/wal
04:01:13.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/ts-1/data/data
04:01:13.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
04:01:13.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
04:01:13.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
04:01:13.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
04:01:13.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
04:01:13.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
04:01:13.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
04:01:13.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
04:01:13.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
04:01:13.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
04:01:13.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
04:01:13.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:13.266788 8360 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
04:01:13.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:13.268242 8360 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
04:01:13.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:13.268635 8360 kserver.cc:163] Server-wide thread pool size limit: 3276
04:01:13.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:13.271214 8360 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
04:01:13.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:13.275358 8360 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
04:01:13.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:13.275547 8360 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
04:01:13.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:13.275749 8360 ts_tablet_manager.cc:610] Registered 0 tablets
04:01:13.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:13.275879 8360 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
04:01:13.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:13.425225 8360 rpc_server.cc:307] RPC server started. Bound to: 127.7.185.194:33817
04:01:13.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:13.425331 8490 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.185.194:33817 every 8 connection(s)
04:01:13.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:13.427683 8360 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/ts-1/data/info.pb
04:01:13.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:13.430545 7911 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskKPlGYQ/build/tsan/bin/kudu as pid 8360
04:01:13.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:13.431349 7911 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/ts-1/wal/instance
04:01:13.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:13.471442 7911 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskKPlGYQ/build/tsan/bin/kudu
04:01:13.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskKPlGYQ/build/tsan/bin/kudu
04:01:13.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/ts-2/wal
04:01:13.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/ts-2/data
04:01:13.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
04:01:13.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
04:01:13.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
04:01:13.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:01:13.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:01:13.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:01:13.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/ts-2/logs
04:01:13.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/ts-2/data/info.pb
04:01:13.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:01:13.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
04:01:13.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
04:01:13.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
04:01:13.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
04:01:13.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:01:13.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:01:13.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:01:13.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
04:01:13.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
04:01:13.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.185.195:0
04:01:13.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.7.185.195
04:01:13.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.185.195
04:01:13.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:01:13.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.7.185.254:35943,127.7.185.253:42165,127.7.185.252:37771
04:01:13.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.185.212:38181
04:01:13.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:01:13.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:01:13.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
04:01:13.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:13.490109 8493 heartbeater.cc:347] Connected to a master server at 127.7.185.254:35943
04:01:13.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:13.490646 8493 heartbeater.cc:464] Registering TS with master...
04:01:13.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:13.491567 8491 heartbeater.cc:347] Connected to a master server at 127.7.185.252:37771
04:01:13.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:13.491922 8491 heartbeater.cc:464] Registering TS with master...
04:01:13.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:13.491959 8493 heartbeater.cc:511] Master 127.7.185.254:35943 requested a full tablet report, sending...
04:01:13.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:13.492784 8491 heartbeater.cc:511] Master 127.7.185.252:37771 requested a full tablet report, sending...
04:01:13.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:13.493542 8492 heartbeater.cc:347] Connected to a master server at 127.7.185.253:42165
04:01:13.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:13.493911 8492 heartbeater.cc:464] Registering TS with master...
04:01:13.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:13.494777 8492 heartbeater.cc:511] Master 127.7.185.253:42165 requested a full tablet report, sending...
04:01:13.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:13.495285 8112 ts_manager.cc:194] Registered new tserver with Master: 1ad0296afe7c432db3797611d1facdc4 (127.7.185.194:33817)
04:01:13.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:13.495867 7956 ts_manager.cc:194] Registered new tserver with Master: 1ad0296afe7c432db3797611d1facdc4 (127.7.185.194:33817)
04:01:13.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:13.497570 8112 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.185.194:41135
04:01:13.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:13.497722 8032 ts_manager.cc:194] Registered new tserver with Master: 1ad0296afe7c432db3797611d1facdc4 (127.7.185.194:33817)
04:01:13.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:13.800419 8501 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
04:01:13.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:13.800911 8501 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
04:01:13.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:13.801355 8501 flags.cc:425] Enabled unsafe flag: --never_fsync=true
04:01:13.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:13.832800 8501 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
04:01:13.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:13.833674 8501 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.185.195
04:01:13.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:13.870396 8501 tablet_server_runner.cc:78] Tablet server non-default flags:
04:01:13.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:01:13.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.185.212:38181
04:01:13.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:01:13.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:01:13.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/ts-2/data
04:01:13.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/ts-2/wal
04:01:13.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:01:13.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:01:13.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.185.195:0
04:01:13.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
04:01:13.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:01:13.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:01:13.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/ts-2/data/info.pb
04:01:13.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.185.195
04:01:13.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:01:13.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.7.185.254:35943,127.7.185.253:42165,127.7.185.252:37771
04:01:13.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
04:01:13.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:01:13.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
04:01:13.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
04:01:13.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:01:13.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.7.185.195
04:01:13.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/ts-2/logs
04:01:13.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:01:13.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
04:01:13.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
04:01:13.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
04:01:13.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
04:01:13.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
04:01:13.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
04:01:13.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 02:58:30 UTC on ccycloud.quasar-xnspdp.root.comops.site
04:01:13.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4569537
04:01:13.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
04:01:13.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:13.871673 8501 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
04:01:13.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:13.873276 8501 file_cache.cc:493] Constructed file cache file cache with capacity 419430
04:01:13.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:13.891983 8507 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:01:14.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:14.502636 8491 heartbeater.cc:503] Master 127.7.185.252:37771 was elected leader, sending a full tablet report...
04:01:15.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:13.892040 8508 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:01:15.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:13.894701 8501 server_base.cc:1034] running on GCE node
04:01:15.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:13.892263 8510 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:01:15.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:15.096678 8501 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
04:01:15.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:15.099573 8501 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
04:01:15.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:15.101009 8501 hybrid_clock.cc:648] HybridClock initialized: now 1785988875100962 us; error 64 us; skew 500 ppm
04:01:15.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:15.101806 8501 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
04:01:15.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:15.108530 8501 webserver.cc:460] Webserver started at http://127.7.185.195:38105/ using document root <none> and password file <none>
04:01:15.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:15.109450 8501 fs_manager.cc:359] Metadata directory not provided
04:01:15.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:15.109649 8501 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
04:01:15.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:15.110193 8501 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
04:01:15.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:15.115218 8501 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/ts-2/data/instance:
04:01:15.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6f849a2f442e4bb2b886d0ba3e68bdaf"
04:01:15.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 04:01:15 on dist-test-slave-1jjb"
04:01:15.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:15.116508 8501 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/ts-2/wal/instance:
04:01:15.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6f849a2f442e4bb2b886d0ba3e68bdaf"
04:01:15.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 04:01:15 on dist-test-slave-1jjb"
04:01:15.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:15.123946 8501 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
04:01:15.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:15.129635 8518 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
04:01:15.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:15.130710 8501 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
04:01:15.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:15.130980 8501 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/ts-2/data,/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/ts-2/wal
04:01:15.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6f849a2f442e4bb2b886d0ba3e68bdaf"
04:01:15.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 04:01:15 on dist-test-slave-1jjb"
04:01:15.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:15.131284 8501 fs_report.cc:383] FS layout report
04:01:15.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
04:01:15.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/ts-2/wal
04:01:15.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/ts-2/wal
04:01:15.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/ts-2/data/data
04:01:15.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
04:01:15.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
04:01:15.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
04:01:15.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
04:01:15.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
04:01:15.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
04:01:15.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
04:01:15.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
04:01:15.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
04:01:15.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
04:01:15.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
04:01:15.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:15.184779 8501 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
04:01:15.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:15.186230 8501 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
04:01:15.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:15.186700 8501 kserver.cc:163] Server-wide thread pool size limit: 3276
04:01:15.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:15.189208 8501 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
04:01:15.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:15.193275 8501 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
04:01:15.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:15.193511 8501 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
04:01:15.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:15.193904 8501 ts_tablet_manager.cc:610] Registered 0 tablets
04:01:15.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:15.194085 8501 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
04:01:15.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:15.341076 8501 rpc_server.cc:307] RPC server started. Bound to: 127.7.185.195:40291
04:01:15.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:15.341193 8631 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.185.195:40291 every 8 connection(s)
04:01:15.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:15.343619 8501 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/ts-2/data/info.pb
04:01:15.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:15.351662 7911 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskKPlGYQ/build/tsan/bin/kudu as pid 8501
04:01:15.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:15.352265 7911 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster7227051504254048254/ts-2/wal/instance
04:01:15.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:15.396924 8633 heartbeater.cc:347] Connected to a master server at 127.7.185.253:42165
04:01:15.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:15.397558 8633 heartbeater.cc:464] Registering TS with master...
04:01:15.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:15.399183 8633 heartbeater.cc:511] Master 127.7.185.253:42165 requested a full tablet report, sending...
04:01:15.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:15.403352 8032 ts_manager.cc:194] Registered new tserver with Master: 6f849a2f442e4bb2b886d0ba3e68bdaf (127.7.185.195:40291)
04:01:15.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:15.419134 8632 heartbeater.cc:347] Connected to a master server at 127.7.185.252:37771
04:01:15.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:15.419260 8634 heartbeater.cc:347] Connected to a master server at 127.7.185.254:35943
04:01:15.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:15.419535 8632 heartbeater.cc:464] Registering TS with master...
04:01:15.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:15.419543 8634 heartbeater.cc:464] Registering TS with master...
04:01:15.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:15.420567 8634 heartbeater.cc:511] Master 127.7.185.254:35943 requested a full tablet report, sending...
04:01:15.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:15.420598 8632 heartbeater.cc:511] Master 127.7.185.252:37771 requested a full tablet report, sending...
04:01:15.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:15.423306 7956 ts_manager.cc:194] Registered new tserver with Master: 6f849a2f442e4bb2b886d0ba3e68bdaf (127.7.185.195:40291)
04:01:15.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:15.423558 8112 ts_manager.cc:194] Registered new tserver with Master: 6f849a2f442e4bb2b886d0ba3e68bdaf (127.7.185.195:40291)
04:01:15.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:15.425261 8112 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.185.195:43585
04:01:15.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:15.443894 7911 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
04:01:15.454 [DEBUG - main] (MiniKuduCluster.java:195) Response:
04:01:15.456 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
04:01:15.467 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.7.185.254"
port: 35943
}
bound_http_address {
host: "127.7.185.254"
port: 42773
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.7.185.253"
port: 42165
}
bound_http_address {
host: "127.7.185.253"
port: 35227
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.7.185.252"
port: 37771
}
bound_http_address {
host: "127.7.185.252"
port: 46405
}
}
}
04:01:15.474 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
04:01:15.477 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.7.185.193"
port: 38495
}
bound_http_address {
host: "127.7.185.193"
port: 38247
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.7.185.194"
port: 33817
}
bound_http_address {
host: "127.7.185.194"
port: 41337
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.7.185.195"
port: 40291
}
bound_http_address {
host: "127.7.185.195"
port: 38105
}
}
}
04:01:15.478 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
04:01:15.654 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
04:01:15.777 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskKPlGYQ/test-tmp
04:01:15.778 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
04:01:15.778 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskKPlGYQ/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
04:01:15.781 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
04:01:16.120 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
04:01:16.120 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
04:01:16.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:16.123873 8643 mini_oidc.cc:152] Starting JWKS server
04:01:16.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:16.127473 8643 webserver.cc:460] Webserver started at http://127.0.0.1:40281/ using document root <none> and password file <none>
04:01:16.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:16.127885 8643 mini_oidc.cc:189] Starting OIDC Discovery server
04:01:16.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:16.129703 8643 webserver.cc:460] Webserver started at http://127.0.0.1:34605/ using document root <none> and password file <none>
04:01:16.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
04:01:16.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/krb5kdc/principal' for realm 'KRBTEST.COM',
04:01:16.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
04:01:16.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 04:01:16 dist-test-slave-1jjb krb5kdc[8656](info): setting up network...
04:01:16.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
04:01:16.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 04:01:16 dist-test-slave-1jjb krb5kdc[8656](info): set up 2 sockets
04:01:16.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 04:01:16 dist-test-slave-1jjb krb5kdc[8656](info): commencing operation
04:01:16.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
04:01:16.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:16.429463 8632 heartbeater.cc:503] Master 127.7.185.252:37771 was elected leader, sending a full tablet report...
04:01:17.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:17.269758 8643 mini_kdc.cc:121] Time spent starting KDC: real 1.114s user 0.002s sys 0.023s
04:01:17.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
04:01:17.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
04:01:17.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
04:01:17.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
04:01:17.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
04:01:17.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
04:01:17.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
04:01:17.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
04:01:17.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
04:01:17.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
04:01:17.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/krb5kdc/test-user.keytab.
04:01:17.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/krb5kdc/test-user.keytab.
04:01:17.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 04:01:17 dist-test-slave-1jjb krb5kdc[8656](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1785988877, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
04:01:17.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
04:01:17.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-06T04:01:17Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
04:01:17.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-06T04:01:17Z Disabled control of system clock
04:01:17.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.8.112.254@KRBTEST.COM; defaulting to no policy
04:01:17.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
04:01:17.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.8.112.254@KRBTEST.COM" created.
04:01:17.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
04:01:17.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.8.112.254 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/krb5kdc/kudu_127.8.112.254.keytab.
04:01:17.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.8.112.254 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/krb5kdc/kudu_127.8.112.254.keytab.
04:01:17.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:17.477988 8643 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskKPlGYQ/build/tsan/bin/kudu
04:01:17.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskKPlGYQ/build/tsan/bin/kudu
04:01:17.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-0/wal
04:01:17.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-0/data
04:01:17.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
04:01:17.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
04:01:17.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
04:01:17.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:01:17.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:01:17.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:01:17.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-0/logs
04:01:17.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-0/data/info.pb
04:01:17.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:01:17.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
04:01:17.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
04:01:17.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
04:01:17.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
04:01:17.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:01:17.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:01:17.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:01:17.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
04:01:17.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
04:01:17.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:01:17.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:01:17.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.112.254:41699
04:01:17.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.112.254
04:01:17.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:01:17.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/krb5kdc/kudu_127.8.112.254.keytab
04:01:17.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.112.254
04:01:17.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
04:01:17.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
04:01:17.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
04:01:17.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.112.212:35587
04:01:17.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:01:17.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:01:17.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:01:17.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:01:17.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.112.254:41699,127.8.112.253:44153,127.8.112.252:42515
04:01:17.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:01:17.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40281/jwks.json
04:01:17.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
04:01:17.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:17.791947 8669 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
04:01:17.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:17.792654 8669 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
04:01:17.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:17.793397 8669 flags.cc:425] Enabled unsafe flag: --never_fsync=true
04:01:17.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:17.829180 8669 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
04:01:17.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:17.829473 8669 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
04:01:17.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:17.829895 8669 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
04:01:17.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:17.830211 8669 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
04:01:17.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:17.830504 8669 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
04:01:17.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:17.830788 8669 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
04:01:17.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:17.831027 8669 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40281/jwks.json
04:01:17.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:17.868376 8669 master_runner.cc:386] Master server non-default flags:
04:01:17.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:01:17.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.112.212:35587
04:01:17.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:01:17.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:01:17.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-0/data
04:01:17.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-0/wal
04:01:17.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
04:01:17.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:01:17.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.112.254:41699,127.8.112.253:44153,127.8.112.252:42515
04:01:17.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/krb5kdc/kudu_127.8.112.254.keytab
04:01:17.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.112.254
04:01:17.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:01:17.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:01:17.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:01:17.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.112.254:41699
04:01:17.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:01:17.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
04:01:17.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:01:17.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40281/jwks.json
04:01:17.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
04:01:17.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
04:01:17.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
04:01:17.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:01:17.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:01:17.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-0/data/info.pb
04:01:17.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.112.254
04:01:17.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:01:17.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
04:01:17.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:01:17.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
04:01:17.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
04:01:17.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:01:17.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-0/logs
04:01:17.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:01:17.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
04:01:17.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
04:01:17.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
04:01:17.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
04:01:17.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
04:01:17.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
04:01:17.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 02:58:30 UTC on ccycloud.quasar-xnspdp.root.comops.site
04:01:17.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4569537
04:01:17.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
04:01:17.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:17.869943 8669 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
04:01:17.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:17.871176 8669 file_cache.cc:493] Constructed file cache file cache with capacity 419430
04:01:17.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:17.888929 8678 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:01:19.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:17.889052 8676 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:01:19.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:17.889542 8675 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:01:19.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:19.134589 8677 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
04:01:19.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:19.134748 8669 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
04:01:19.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:19.138916 8669 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
04:01:19.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:19.142225 8669 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
04:01:19.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:19.143721 8669 hybrid_clock.cc:648] HybridClock initialized: now 1785988879143708 us; error 132 us; skew 500 ppm
04:01:19.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:19.144574 8669 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
04:01:19.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 04:01:19 dist-test-slave-1jjb krb5kdc[8656](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1785988879, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.112.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
04:01:19.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:19.150568 8669 init.cc:377] Logged in from keytab as kudu/127.8.112.254@KRBTEST.COM (short username kudu)
04:01:19.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:19.155791 8669 webserver.cc:460] Webserver started at http://127.8.112.254:43437/ using document root <none> and password file <none>
04:01:19.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:19.156858 8669 fs_manager.cc:359] Metadata directory not provided
04:01:19.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:19.157047 8669 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
04:01:19.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:19.157442 8669 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
04:01:19.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:19.162241 8669 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-0/data/instance:
04:01:19.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7e5cffbbe0ff4b0ca459bb5fd221108b"
04:01:19.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 04:01:19 on dist-test-slave-1jjb"
04:01:19.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:19.163298 8669 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-0/wal/instance:
04:01:19.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7e5cffbbe0ff4b0ca459bb5fd221108b"
04:01:19.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 04:01:19 on dist-test-slave-1jjb"
04:01:19.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:19.170760 8669 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
04:01:19.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:19.176134 8686 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
04:01:19.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:19.177211 8669 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
04:01:19.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:19.177500 8669 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-0/data,/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-0/wal
04:01:19.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7e5cffbbe0ff4b0ca459bb5fd221108b"
04:01:19.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 04:01:19 on dist-test-slave-1jjb"
04:01:19.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:19.177804 8669 fs_report.cc:383] FS layout report
04:01:19.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
04:01:19.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-0/wal
04:01:19.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-0/wal
04:01:19.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-0/data/data
04:01:19.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
04:01:19.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
04:01:19.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
04:01:19.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
04:01:19.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
04:01:19.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
04:01:19.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
04:01:19.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
04:01:19.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
04:01:19.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
04:01:19.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
04:01:19.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:19.218775 8669 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
04:01:19.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:19.224375 8669 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
04:01:19.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:19.224782 8669 kserver.cc:163] Server-wide thread pool size limit: 3276
04:01:19.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:19.301144 8669 rpc_server.cc:307] RPC server started. Bound to: 127.8.112.254:41699
04:01:19.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:19.301242 8738 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.8.112.254:41699 every 8 connection(s)
04:01:19.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:19.303824 8669 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-0/data/info.pb
04:01:19.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:19.309116 8739 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
04:01:19.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:19.312585 8643 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskKPlGYQ/build/tsan/bin/kudu as pid 8669
04:01:19.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:19.312969 8643 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-0/wal/instance
04:01:19.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:19.321342 8739 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.112.254" port: 41699 } has no permanent_uuid. Determining permanent_uuid...
04:01:19.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.8.112.253@KRBTEST.COM; defaulting to no policy
04:01:19.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
04:01:19.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.8.112.253@KRBTEST.COM" created.
04:01:19.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 04:01:19 dist-test-slave-1jjb krb5kdc[8656](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1785988879, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.112.254@KRBTEST.COM for kudu/127.8.112.254@KRBTEST.COM
04:01:19.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
04:01:19.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.8.112.253 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/krb5kdc/kudu_127.8.112.253.keytab.
04:01:19.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.8.112.253 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/krb5kdc/kudu_127.8.112.253.keytab.
04:01:19.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:19.368148 8739 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.112.253" port: 44153 } has no permanent_uuid. Determining permanent_uuid...
04:01:19.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:19.369123 8643 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskKPlGYQ/build/tsan/bin/kudu
04:01:19.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskKPlGYQ/build/tsan/bin/kudu
04:01:19.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-1/wal
04:01:19.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-1/data
04:01:19.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
04:01:19.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
04:01:19.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
04:01:19.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:01:19.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:01:19.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:01:19.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-1/logs
04:01:19.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-1/data/info.pb
04:01:19.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:01:19.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
04:01:19.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
04:01:19.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
04:01:19.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
04:01:19.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:01:19.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:01:19.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:01:19.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
04:01:19.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
04:01:19.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:01:19.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:01:19.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.112.253:44153
04:01:19.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.112.253
04:01:19.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:01:19.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/krb5kdc/kudu_127.8.112.253.keytab
04:01:19.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.112.253
04:01:19.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
04:01:19.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
04:01:19.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
04:01:19.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.112.212:35587
04:01:19.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:01:19.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:01:19.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:01:19.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:01:19.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.112.254:41699,127.8.112.253:44153,127.8.112.252:42515
04:01:19.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:01:19.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40281/jwks.json
04:01:19.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
04:01:19.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:19.371176 8689 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.112.253:44153: connect: Connection refused (error 111)
04:01:19.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:19.375671 8739 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.112.253:44153: Network error: Client connection negotiation failed: client connection to 127.8.112.253:44153: connect: Connection refused (error 111)
04:01:19.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:19.425161 8739 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.112.253" port: 44153 } attempt: 1
04:01:19.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:19.429257 8739 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.112.253:44153: Network error: Client connection negotiation failed: client connection to 127.8.112.253:44153: connect: Connection refused (error 111)
04:01:19.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:19.497686 8739 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.112.253" port: 44153 } attempt: 2
04:01:19.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:19.501683 8739 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.112.253:44153: Network error: Client connection negotiation failed: client connection to 127.8.112.253:44153: connect: Connection refused (error 111)
04:01:19.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:19.593088 8739 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.112.253" port: 44153 } attempt: 3
04:01:19.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:19.597110 8739 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.112.253:44153: Network error: Client connection negotiation failed: client connection to 127.8.112.253:44153: connect: Connection refused (error 111)
04:01:19.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:19.705854 8746 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
04:01:19.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:19.706405 8746 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
04:01:19.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:19.707010 8746 flags.cc:425] Enabled unsafe flag: --never_fsync=true
04:01:19.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:19.740562 8739 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.112.253" port: 44153 } attempt: 4
04:01:19.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:19.741366 8746 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
04:01:19.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:19.741668 8746 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
04:01:19.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:19.742340 8746 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
04:01:19.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:19.742746 8746 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
04:01:19.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:19.743212 8746 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
04:01:19.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:19.743468 8746 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
04:01:19.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:19.743614 8746 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40281/jwks.json
04:01:19.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:19.744865 8739 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.112.253:44153: Network error: Client connection negotiation failed: client connection to 127.8.112.253:44153: connect: Connection refused (error 111)
04:01:19.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:19.782972 8746 master_runner.cc:386] Master server non-default flags:
04:01:19.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:01:19.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.112.212:35587
04:01:19.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:01:19.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:01:19.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-1/data
04:01:19.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-1/wal
04:01:19.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
04:01:19.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:01:19.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.112.254:41699,127.8.112.253:44153,127.8.112.252:42515
04:01:19.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/krb5kdc/kudu_127.8.112.253.keytab
04:01:19.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.112.253
04:01:19.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:01:19.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:01:19.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:01:19.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.112.253:44153
04:01:19.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:01:19.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
04:01:19.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:01:19.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40281/jwks.json
04:01:19.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
04:01:19.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
04:01:19.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
04:01:19.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:01:19.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:01:19.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-1/data/info.pb
04:01:19.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.112.253
04:01:19.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:01:19.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
04:01:19.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:01:19.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
04:01:19.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
04:01:19.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:01:19.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-1/logs
04:01:19.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:01:19.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
04:01:19.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
04:01:19.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
04:01:19.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
04:01:19.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
04:01:19.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
04:01:19.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 02:58:30 UTC on ccycloud.quasar-xnspdp.root.comops.site
04:01:19.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4569537
04:01:19.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
04:01:19.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:19.784237 8746 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
04:01:19.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:19.785305 8746 file_cache.cc:493] Constructed file cache file cache with capacity 419430
04:01:19.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:19.802894 8752 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:01:20.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:20.044305 8739 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.112.253" port: 44153 } attempt: 5
04:01:20.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:20.047936 8739 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.112.253:44153: Network error: Client connection negotiation failed: client connection to 127.8.112.253:44153: connect: Connection refused (error 111)
04:01:20.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:20.595587 8739 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.112.253" port: 44153 } attempt: 6
04:01:20.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:20.603025 8739 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.112.253:44153: Network error: Client connection negotiation failed: client connection to 127.8.112.253:44153: connect: Connection refused (error 111)
04:01:21.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:19.804847 8746 server_base.cc:1034] running on GCE node
04:01:21.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:19.805713 8755 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:01:21.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:19.803064 8753 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:01:21.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:21.034641 8746 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
04:01:21.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:21.037526 8746 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
04:01:21.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:21.041039 8746 hybrid_clock.cc:648] HybridClock initialized: now 1785988881039868 us; error 1171 us; skew 500 ppm
04:01:21.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:21.041939 8746 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
04:01:21.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 04:01:21 dist-test-slave-1jjb krb5kdc[8656](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1785988881, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.112.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
04:01:21.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:21.048926 8746 init.cc:377] Logged in from keytab as kudu/127.8.112.253@KRBTEST.COM (short username kudu)
04:01:21.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:21.054504 8746 webserver.cc:460] Webserver started at http://127.8.112.253:35951/ using document root <none> and password file <none>
04:01:21.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:21.055428 8746 fs_manager.cc:359] Metadata directory not provided
04:01:21.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:21.055647 8746 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
04:01:21.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:21.056034 8746 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
04:01:21.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:21.061120 8746 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-1/data/instance:
04:01:21.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d556530449f84860bc4c8edc9e606fd9"
04:01:21.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 04:01:21 on dist-test-slave-1jjb"
04:01:21.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:21.062202 8746 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-1/wal/instance:
04:01:21.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d556530449f84860bc4c8edc9e606fd9"
04:01:21.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 04:01:21 on dist-test-slave-1jjb"
04:01:21.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:21.070206 8746 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.004s sys 0.004s
04:01:21.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:21.075845 8765 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
04:01:21.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:21.076956 8746 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
04:01:21.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:21.077239 8746 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-1/data,/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-1/wal
04:01:21.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d556530449f84860bc4c8edc9e606fd9"
04:01:21.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 04:01:21 on dist-test-slave-1jjb"
04:01:21.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:21.077545 8746 fs_report.cc:383] FS layout report
04:01:21.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
04:01:21.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-1/wal
04:01:21.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-1/wal
04:01:21.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-1/data/data
04:01:21.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
04:01:21.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
04:01:21.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
04:01:21.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
04:01:21.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
04:01:21.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
04:01:21.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
04:01:21.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
04:01:21.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
04:01:21.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
04:01:21.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
04:01:21.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:21.111845 8746 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
04:01:21.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:21.117947 8746 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
04:01:21.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:21.118355 8746 kserver.cc:163] Server-wide thread pool size limit: 3276
04:01:21.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:21.192838 8746 rpc_server.cc:307] RPC server started. Bound to: 127.8.112.253:44153
04:01:21.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:21.193019 8817 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.8.112.253:44153 every 8 connection(s)
04:01:21.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:21.195587 8746 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-1/data/info.pb
04:01:21.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:21.201538 8818 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
04:01:21.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:21.202131 8643 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskKPlGYQ/build/tsan/bin/kudu as pid 8746
04:01:21.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:21.202454 8643 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-1/wal/instance
04:01:21.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:21.214179 8818 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.112.254" port: 41699 } has no permanent_uuid. Determining permanent_uuid...
04:01:21.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.8.112.252@KRBTEST.COM; defaulting to no policy
04:01:21.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
04:01:21.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.8.112.252@KRBTEST.COM" created.
04:01:21.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 04:01:21 dist-test-slave-1jjb krb5kdc[8656](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1785988881, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.112.253@KRBTEST.COM for kudu/127.8.112.254@KRBTEST.COM
04:01:21.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
04:01:21.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.8.112.252 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/krb5kdc/kudu_127.8.112.252.keytab.
04:01:21.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.8.112.252 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/krb5kdc/kudu_127.8.112.252.keytab.
04:01:21.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:21.252971 8643 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskKPlGYQ/build/tsan/bin/kudu
04:01:21.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskKPlGYQ/build/tsan/bin/kudu
04:01:21.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-2/wal
04:01:21.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-2/data
04:01:21.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
04:01:21.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
04:01:21.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
04:01:21.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:01:21.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:01:21.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:01:21.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-2/logs
04:01:21.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-2/data/info.pb
04:01:21.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:01:21.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
04:01:21.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
04:01:21.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
04:01:21.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
04:01:21.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:01:21.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:01:21.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:01:21.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
04:01:21.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
04:01:21.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:01:21.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:01:21.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.112.252:42515
04:01:21.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.112.252
04:01:21.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:01:21.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/krb5kdc/kudu_127.8.112.252.keytab
04:01:21.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.112.252
04:01:21.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
04:01:21.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
04:01:21.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
04:01:21.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.112.212:35587
04:01:21.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:01:21.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:01:21.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:01:21.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:01:21.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.112.254:41699,127.8.112.253:44153,127.8.112.252:42515
04:01:21.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:01:21.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40281/jwks.json
04:01:21.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
04:01:21.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:21.257779 8818 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.112.253" port: 44153 } has no permanent_uuid. Determining permanent_uuid...
04:01:21.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 04:01:21 dist-test-slave-1jjb krb5kdc[8656](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1785988881, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.112.253@KRBTEST.COM for kudu/127.8.112.253@KRBTEST.COM
04:01:21.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:21.288535 8818 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.112.252" port: 42515 } has no permanent_uuid. Determining permanent_uuid...
04:01:21.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:21.290805 8768 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.112.252:42515: connect: Connection refused (error 111)
04:01:21.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:21.294955 8818 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.112.252:42515: Network error: Client connection negotiation failed: client connection to 127.8.112.252:42515: connect: Connection refused (error 111)
04:01:21.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:21.344424 8818 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.112.252" port: 42515 } attempt: 1
04:01:21.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:21.349316 8818 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.112.252:42515: Network error: Client connection negotiation failed: client connection to 127.8.112.252:42515: connect: Connection refused (error 111)
04:01:21.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:21.417817 8818 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.112.252" port: 42515 } attempt: 2
04:01:21.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:21.422039 8818 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.112.252:42515: Network error: Client connection negotiation failed: client connection to 127.8.112.252:42515: connect: Connection refused (error 111)
04:01:21.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:21.513528 8818 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.112.252" port: 42515 } attempt: 3
04:01:21.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:21.517541 8818 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.112.252:42515: Network error: Client connection negotiation failed: client connection to 127.8.112.252:42515: connect: Connection refused (error 111)
04:01:21.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:21.593147 8824 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
04:01:21.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:21.593711 8824 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
04:01:21.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:21.594429 8824 flags.cc:425] Enabled unsafe flag: --never_fsync=true
04:01:21.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:21.625917 8824 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
04:01:21.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:21.626204 8824 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
04:01:21.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:21.626482 8824 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
04:01:21.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:21.626665 8824 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
04:01:21.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:21.626852 8824 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
04:01:21.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:21.627036 8824 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
04:01:21.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:21.627172 8824 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40281/jwks.json
04:01:21.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:21.660989 8818 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.112.252" port: 42515 } attempt: 4
04:01:21.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:21.663616 8739 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.112.253" port: 44153 } attempt: 7
04:01:21.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:21.664295 8824 master_runner.cc:386] Master server non-default flags:
04:01:21.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:01:21.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.112.212:35587
04:01:21.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:01:21.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:01:21.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-2/data
04:01:21.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-2/wal
04:01:21.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
04:01:21.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:01:21.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.112.254:41699,127.8.112.253:44153,127.8.112.252:42515
04:01:21.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/krb5kdc/kudu_127.8.112.252.keytab
04:01:21.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.112.252
04:01:21.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:01:21.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:01:21.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:01:21.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.112.252:42515
04:01:21.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:01:21.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
04:01:21.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:01:21.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40281/jwks.json
04:01:21.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
04:01:21.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
04:01:21.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
04:01:21.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:01:21.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:01:21.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-2/data/info.pb
04:01:21.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.112.252
04:01:21.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:01:21.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
04:01:21.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:01:21.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
04:01:21.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
04:01:21.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:01:21.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-2/logs
04:01:21.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:01:21.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
04:01:21.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
04:01:21.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
04:01:21.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
04:01:21.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
04:01:21.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
04:01:21.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 02:58:30 UTC on ccycloud.quasar-xnspdp.root.comops.site
04:01:21.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4569537
04:01:21.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
04:01:21.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:21.665474 8818 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.112.252:42515: Network error: Client connection negotiation failed: client connection to 127.8.112.252:42515: connect: Connection refused (error 111)
04:01:21.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:21.665961 8824 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
04:01:21.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:21.667559 8824 file_cache.cc:493] Constructed file cache file cache with capacity 419430
04:01:21.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 04:01:21 dist-test-slave-1jjb krb5kdc[8656](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1785988879, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.112.254@KRBTEST.COM for kudu/127.8.112.253@KRBTEST.COM
04:01:21.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:21.689085 8836 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:01:21.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:21.697811 8739 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.112.252" port: 42515 } has no permanent_uuid. Determining permanent_uuid...
04:01:21.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:21.703799 8739 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.112.252:42515: Network error: Client connection negotiation failed: client connection to 127.8.112.252:42515: connect: Connection refused (error 111)
04:01:21.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:21.762257 8739 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.112.252" port: 42515 } attempt: 1
04:01:21.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:21.766597 8739 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.112.252:42515: Network error: Client connection negotiation failed: client connection to 127.8.112.252:42515: connect: Connection refused (error 111)
04:01:21.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:21.848273 8739 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.112.252" port: 42515 } attempt: 2
04:01:21.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:21.855114 8739 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.112.252:42515: Network error: Client connection negotiation failed: client connection to 127.8.112.252:42515: connect: Connection refused (error 111)
04:01:21.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:21.940636 8739 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.112.252" port: 42515 } attempt: 3
04:01:21.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:21.944281 8739 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.112.252:42515: Network error: Client connection negotiation failed: client connection to 127.8.112.252:42515: connect: Connection refused (error 111)
04:01:21.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:21.964898 8818 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.112.252" port: 42515 } attempt: 5
04:01:21.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:21.968525 8818 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.112.252:42515: Network error: Client connection negotiation failed: client connection to 127.8.112.252:42515: connect: Connection refused (error 111)
04:01:22.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:22.084933 8739 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.112.252" port: 42515 } attempt: 4
04:01:22.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:22.089233 8739 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.112.252:42515: Network error: Client connection negotiation failed: client connection to 127.8.112.252:42515: connect: Connection refused (error 111)
04:01:22.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:22.372884 8739 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.112.252" port: 42515 } attempt: 5
04:01:22.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:22.377322 8739 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.112.252:42515: Network error: Client connection negotiation failed: client connection to 127.8.112.252:42515: connect: Connection refused (error 111)
04:01:22.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:22.516196 8818 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.112.252" port: 42515 } attempt: 6
04:01:22.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:22.523926 8818 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.112.252:42515: Network error: Client connection negotiation failed: client connection to 127.8.112.252:42515: connect: Connection refused (error 111)
04:01:22.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:22.929931 8739 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.112.252" port: 42515 } attempt: 6
04:01:22.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:22.937252 8739 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.112.252:42515: Network error: Client connection negotiation failed: client connection to 127.8.112.252:42515: connect: Connection refused (error 111)
04:01:22.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:21.692520 8833 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:01:22.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:21.689172 8834 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:01:22.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:22.943969 8835 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
04:01:22.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:22.944003 8824 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
04:01:22.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:22.948344 8824 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
04:01:22.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:22.951332 8824 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
04:01:22.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:22.952718 8824 hybrid_clock.cc:648] HybridClock initialized: now 1785988882952672 us; error 49 us; skew 500 ppm
04:01:22.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:22.953560 8824 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
04:01:22.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 04:01:22 dist-test-slave-1jjb krb5kdc[8656](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1785988882, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.112.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
04:01:22.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:22.959630 8824 init.cc:377] Logged in from keytab as kudu/127.8.112.252@KRBTEST.COM (short username kudu)
04:01:22.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:22.965173 8824 webserver.cc:460] Webserver started at http://127.8.112.252:46415/ using document root <none> and password file <none>
04:01:22.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:22.966138 8824 fs_manager.cc:359] Metadata directory not provided
04:01:22.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:22.966380 8824 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
04:01:22.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:22.966902 8824 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
04:01:22.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:22.972151 8824 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-2/data/instance:
04:01:22.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0c56e39950cc4685bfaff0cfc2d7b48d"
04:01:22.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 04:01:22 on dist-test-slave-1jjb"
04:01:22.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:22.973385 8824 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-2/wal/instance:
04:01:22.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0c56e39950cc4685bfaff0cfc2d7b48d"
04:01:22.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 04:01:22 on dist-test-slave-1jjb"
04:01:22.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:22.981223 8824 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
04:01:22.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:22.987047 8853 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
04:01:22.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:22.988169 8824 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
04:01:22.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:22.988534 8824 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-2/data,/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-2/wal
04:01:22.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0c56e39950cc4685bfaff0cfc2d7b48d"
04:01:22.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 04:01:22 on dist-test-slave-1jjb"
04:01:22.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:22.988880 8824 fs_report.cc:383] FS layout report
04:01:22.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
04:01:22.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-2/wal
04:01:22.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-2/wal
04:01:22.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-2/data/data
04:01:22.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
04:01:22.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
04:01:22.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
04:01:22.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
04:01:22.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
04:01:22.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
04:01:22.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
04:01:22.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
04:01:22.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
04:01:22.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
04:01:22.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
04:01:23.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:23.026291 8824 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
04:01:23.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:23.034440 8824 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
04:01:23.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:23.035014 8824 kserver.cc:163] Server-wide thread pool size limit: 3276
04:01:23.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:23.119879 8824 rpc_server.cc:307] RPC server started. Bound to: 127.8.112.252:42515
04:01:23.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:23.119957 8906 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.8.112.252:42515 every 8 connection(s)
04:01:23.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:23.122633 8824 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-2/data/info.pb
04:01:23.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:23.123209 8643 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskKPlGYQ/build/tsan/bin/kudu as pid 8824
04:01:23.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:23.123556 8643 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-2/wal/instance
04:01:23.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:23.132640 8907 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
04:01:23.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:23.148859 8907 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.112.254" port: 41699 } has no permanent_uuid. Determining permanent_uuid...
04:01:23.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 04:01:23 dist-test-slave-1jjb krb5kdc[8656](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1785988877, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.8.112.252@KRBTEST.COM
04:01:23.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 04:01:23 dist-test-slave-1jjb krb5kdc[8656](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1785988882, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.112.252@KRBTEST.COM for kudu/127.8.112.254@KRBTEST.COM
04:01:23.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:23.180243 8907 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.112.253" port: 44153 } has no permanent_uuid. Determining permanent_uuid...
04:01:23.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 04:01:23 dist-test-slave-1jjb krb5kdc[8656](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1785988882, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.112.252@KRBTEST.COM for kudu/127.8.112.253@KRBTEST.COM
04:01:23.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 04:01:23 dist-test-slave-1jjb krb5kdc[8656](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1785988877, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.8.112.253@KRBTEST.COM
04:01:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:23.215934 8907 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.112.252" port: 42515 } has no permanent_uuid. Determining permanent_uuid...
04:01:23.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 04:01:23 dist-test-slave-1jjb krb5kdc[8656](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1785988882, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.112.252@KRBTEST.COM for kudu/127.8.112.252@KRBTEST.COM
04:01:23.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 04:01:23 dist-test-slave-1jjb krb5kdc[8656](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1785988877, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.8.112.254@KRBTEST.COM
04:01:23.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:23.248191 8643 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
04:01:23.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:23.248574 8907 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d: Bootstrap starting.
04:01:23.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:23.255229 8907 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d: Neither blocks nor log segments found. Creating new log.
04:01:23.257 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
04:01:23.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:23.257207 8907 log.cc:824] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d: Log is configured to *not* fsync() on all Append() calls
04:01:23.258 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
04:01:23.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:23.262230 8907 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d: No bootstrap required, opened a new log
04:01:23.268 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
04:01:23.269 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
04:01:23.272 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.8.112.254"
port: 41699
}
bound_http_address {
host: "127.8.112.254"
port: 43437
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.8.112.253"
port: 44153
}
bound_http_address {
host: "127.8.112.253"
port: 35951
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.8.112.252"
port: 42515
}
bound_http_address {
host: "127.8.112.252"
port: 46415
}
}
}
04:01:23.273 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
04:01:23.273 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
04:01:23.273 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
04:01:23.275 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
04:01:23.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:23.285620 8907 raft_consensus.cc:348] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7e5cffbbe0ff4b0ca459bb5fd221108b" member_type: VOTER last_known_addr { host: "127.8.112.254" port: 41699 } } peers { permanent_uuid: "d556530449f84860bc4c8edc9e606fd9" member_type: VOTER last_known_addr { host: "127.8.112.253" port: 44153 } } peers { permanent_uuid: "0c56e39950cc4685bfaff0cfc2d7b48d" member_type: VOTER last_known_addr { host: "127.8.112.252" port: 42515 } }
04:01:23.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:23.286424 8907 raft_consensus.cc:374] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
04:01:23.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:23.286697 8907 raft_consensus.cc:729] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0c56e39950cc4685bfaff0cfc2d7b48d, State: Initialized, Role: FOLLOWER
04:01:23.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:23.287637 8907 consensus_queue.cc:260] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7e5cffbbe0ff4b0ca459bb5fd221108b" member_type: VOTER last_known_addr { host: "127.8.112.254" port: 41699 } } peers { permanent_uuid: "d556530449f84860bc4c8edc9e606fd9" member_type: VOTER last_known_addr { host: "127.8.112.253" port: 44153 } } peers { permanent_uuid: "0c56e39950cc4685bfaff0cfc2d7b48d" member_type: VOTER last_known_addr { host: "127.8.112.252" port: 42515 } }
04:01:23.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:23.290503 8917 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7e5cffbbe0ff4b0ca459bb5fd221108b" member_type: VOTER last_known_addr { host: "127.8.112.254" port: 41699 } } peers { permanent_uuid: "d556530449f84860bc4c8edc9e606fd9" member_type: VOTER last_known_addr { host: "127.8.112.253" port: 44153 } } peers { permanent_uuid: "0c56e39950cc4685bfaff0cfc2d7b48d" member_type: VOTER last_known_addr { host: "127.8.112.252" port: 42515 } } }
04:01:23.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:23.291337 8860 tablet.cc:2369] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d: Can't schedule compaction. Clean time has not been advanced past its initial value.
04:01:23.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:23.291540 8917 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d [sys.catalog]: This master's current role is: FOLLOWER
04:01:23.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:23.291956 8907 sys_catalog.cc:564] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d [sys.catalog]: configured and running, proceeding with master startup.
04:01:23.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 04:01:23 dist-test-slave-1jjb krb5kdc[8656](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1785988883, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
04:01:23.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
04:01:23.329 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
04:01:23.332 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/krb5kdc/krb5cc
04:01:23.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:23.330628 8929 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:01:23.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:23.331056 8929 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:01:23.341 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
04:01:23.342 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/krb5kdc/krb5cc
04:01:23.344 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
04:01:23.349 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
04:01:23.351 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
04:01:23.351 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
04:01:23.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:23.320935 8824 master_runner.cc:186] Error getting master registration for 127.8.112.254:41699: OK, instance_id { permanent_uuid: "7e5cffbbe0ff4b0ca459bb5fd221108b" instance_seqno: 1785988879281742 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
04:01:23.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:23.355711 8824 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
04:01:23.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:23.379644 8917 raft_consensus.cc:482] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
04:01:23.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:23.380219 8917 raft_consensus.cc:504] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7e5cffbbe0ff4b0ca459bb5fd221108b" member_type: VOTER last_known_addr { host: "127.8.112.254" port: 41699 } } peers { permanent_uuid: "d556530449f84860bc4c8edc9e606fd9" member_type: VOTER last_known_addr { host: "127.8.112.253" port: 44153 } } peers { permanent_uuid: "0c56e39950cc4685bfaff0cfc2d7b48d" member_type: VOTER last_known_addr { host: "127.8.112.252" port: 42515 } }
04:01:23.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:23.382648 8917 leader_election.cc:288] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7e5cffbbe0ff4b0ca459bb5fd221108b (127.8.112.254:41699), d556530449f84860bc4c8edc9e606fd9 (127.8.112.253:44153)
04:01:23.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 04:01:23 dist-test-slave-1jjb krb5kdc[8656](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1785988883, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
04:01:23.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
04:01:23.401 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
04:01:23.402 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/krb5kdc/krb5cc
04:01:23.404 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
04:01:23.404 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.8.112.253:44153
04:01:23.406 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
04:01:23.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:23.407793 8643 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskKPlGYQ/build/tsan/bin/kudu with pid 8746
04:01:23.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:23.438491 8856 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv got EOF from 127.8.112.253:44153 (error 108)
04:01:23.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:23.447892 8856 leader_election.cc:334] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d556530449f84860bc4c8edc9e606fd9 (127.8.112.253:44153): Network error: Client connection negotiation failed: client connection to 127.8.112.253:44153: connect: Connection refused (error 111)
04:01:23.449 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
04:01:23.449 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.8.112.254:41699
04:01:23.450 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
04:01:23.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:23.450735 8643 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskKPlGYQ/build/tsan/bin/kudu with pid 8669
04:01:23.488 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
04:01:23.488 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.8.112.252:42515
04:01:23.489 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
04:01:23.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:23.489980 8643 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskKPlGYQ/build/tsan/bin/kudu with pid 8824
04:01:23.516 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
04:01:23.516 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.8.112.253:44153
04:01:23.518 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
04:01:23.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:23.519923 8643 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskKPlGYQ/build/tsan/bin/kudu
04:01:23.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskKPlGYQ/build/tsan/bin/kudu
04:01:23.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-1/wal
04:01:23.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-1/data
04:01:23.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
04:01:23.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
04:01:23.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
04:01:23.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:01:23.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:01:23.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:01:23.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-1/logs
04:01:23.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-1/data/info.pb
04:01:23.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:01:23.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
04:01:23.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
04:01:23.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
04:01:23.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
04:01:23.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:01:23.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:01:23.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:01:23.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
04:01:23.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
04:01:23.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:01:23.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:01:23.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.112.253:44153
04:01:23.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.112.253
04:01:23.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35951
04:01:23.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/krb5kdc/kudu_127.8.112.253.keytab
04:01:23.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.112.253
04:01:23.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
04:01:23.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
04:01:23.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
04:01:23.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.112.212:35587
04:01:23.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:01:23.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:01:23.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:01:23.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:01:23.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.112.254:41699,127.8.112.253:44153,127.8.112.252:42515
04:01:23.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:01:23.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40281/jwks.json
04:01:23.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
04:01:23.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:23.844391 8934 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
04:01:23.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:23.844945 8934 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
04:01:23.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:23.845537 8934 flags.cc:425] Enabled unsafe flag: --never_fsync=true
04:01:23.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:23.879290 8934 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
04:01:23.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:23.879529 8934 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
04:01:23.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:23.879807 8934 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
04:01:23.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:23.880013 8934 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
04:01:23.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:23.880328 8934 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
04:01:23.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:23.880524 8934 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
04:01:23.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:23.880651 8934 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40281/jwks.json
04:01:23.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:23.919395 8934 master_runner.cc:386] Master server non-default flags:
04:01:23.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:01:23.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.112.212:35587
04:01:23.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:01:23.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:01:23.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-1/data
04:01:23.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-1/wal
04:01:23.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
04:01:23.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:01:23.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.112.254:41699,127.8.112.253:44153,127.8.112.252:42515
04:01:23.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/krb5kdc/kudu_127.8.112.253.keytab
04:01:23.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.112.253
04:01:23.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:01:23.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:01:23.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:01:23.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.112.253:44153
04:01:23.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:01:23.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
04:01:23.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:01:23.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40281/jwks.json
04:01:23.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
04:01:23.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
04:01:23.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
04:01:23.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:01:23.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:01:23.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-1/data/info.pb
04:01:23.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.112.253
04:01:23.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35951
04:01:23.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
04:01:23.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:01:23.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
04:01:23.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
04:01:23.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:01:23.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-1/logs
04:01:23.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:01:23.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
04:01:23.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
04:01:23.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
04:01:23.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
04:01:23.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
04:01:23.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
04:01:23.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 02:58:30 UTC on ccycloud.quasar-xnspdp.root.comops.site
04:01:23.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4569537
04:01:23.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
04:01:23.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:23.920627 8934 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
04:01:23.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:23.921598 8934 file_cache.cc:493] Constructed file cache file cache with capacity 419430
04:01:23.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:23.940579 8942 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:01:25.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:23.940676 8944 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:01:25.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:23.942384 8934 server_base.cc:1034] running on GCE node
04:01:25.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:23.940953 8941 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:01:25.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:25.176631 8934 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
04:01:25.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:25.179666 8934 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
04:01:25.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:25.181084 8934 hybrid_clock.cc:648] HybridClock initialized: now 1785988885181005 us; error 96 us; skew 500 ppm
04:01:25.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:25.181913 8934 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
04:01:25.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 04:01:25 dist-test-slave-1jjb krb5kdc[8656](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1785988885, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.112.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
04:01:25.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:25.188045 8934 init.cc:377] Logged in from keytab as kudu/127.8.112.253@KRBTEST.COM (short username kudu)
04:01:25.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:25.193423 8934 webserver.cc:460] Webserver started at http://127.8.112.253:35951/ using document root <none> and password file <none>
04:01:25.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:25.194545 8934 fs_manager.cc:359] Metadata directory not provided
04:01:25.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:25.194804 8934 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
04:01:25.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:25.203222 8934 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.005s sys 0.000s
04:01:25.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:25.207726 8953 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
04:01:25.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:25.208904 8934 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.000s sys 0.003s
04:01:25.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:25.209285 8934 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-1/data,/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-1/wal
04:01:25.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d556530449f84860bc4c8edc9e606fd9"
04:01:25.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 04:01:21 on dist-test-slave-1jjb"
04:01:25.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:25.211874 8934 fs_report.cc:383] FS layout report
04:01:25.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
04:01:25.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-1/wal
04:01:25.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-1/wal
04:01:25.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-1/data/data
04:01:25.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
04:01:25.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
04:01:25.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
04:01:25.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
04:01:25.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
04:01:25.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
04:01:25.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
04:01:25.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
04:01:25.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
04:01:25.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
04:01:25.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
04:01:25.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:25.258550 8934 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
04:01:25.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:25.264288 8934 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
04:01:25.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:25.264703 8934 kserver.cc:163] Server-wide thread pool size limit: 3276
04:01:25.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:25.339555 8934 rpc_server.cc:307] RPC server started. Bound to: 127.8.112.253:44153
04:01:25.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:25.339663 9006 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.8.112.253:44153 every 8 connection(s)
04:01:25.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:25.343377 8934 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-1/data/info.pb
04:01:25.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:25.347891 8643 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskKPlGYQ/build/tsan/bin/kudu as pid 8934
04:01:25.348 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
04:01:25.348 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.8.112.254:41699
04:01:25.349 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
04:01:25.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:25.350919 8643 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskKPlGYQ/build/tsan/bin/kudu
04:01:25.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskKPlGYQ/build/tsan/bin/kudu
04:01:25.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-0/wal
04:01:25.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-0/data
04:01:25.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
04:01:25.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
04:01:25.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
04:01:25.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:01:25.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:01:25.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:01:25.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-0/logs
04:01:25.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-0/data/info.pb
04:01:25.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:01:25.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
04:01:25.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
04:01:25.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
04:01:25.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
04:01:25.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:01:25.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:01:25.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:01:25.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
04:01:25.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
04:01:25.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:01:25.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:01:25.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.112.254:41699
04:01:25.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.112.254
04:01:25.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43437
04:01:25.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/krb5kdc/kudu_127.8.112.254.keytab
04:01:25.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.112.254
04:01:25.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
04:01:25.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
04:01:25.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
04:01:25.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.112.212:35587
04:01:25.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:01:25.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:01:25.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:01:25.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:01:25.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.112.254:41699,127.8.112.253:44153,127.8.112.252:42515
04:01:25.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:01:25.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40281/jwks.json
04:01:25.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
04:01:25.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:25.358769 9007 sys_catalog.cc:263] Verifying existing consensus state
04:01:25.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260806 04:01:25.359485 9007 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
04:01:25.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:25.359879 8934 master.cc:537] Master@127.8.112.253:44153 shutting down...
04:01:25.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:25.440246 8934 master.cc:552] Master@127.8.112.253:44153 shutdown complete.
04:01:25.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
04:01:25.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:25.690871 9009 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
04:01:25.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:25.691594 9009 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
04:01:25.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:25.692420 9009 flags.cc:425] Enabled unsafe flag: --never_fsync=true
04:01:25.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:25.724252 9009 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
04:01:25.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:25.724543 9009 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
04:01:25.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:25.724964 9009 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
04:01:25.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:25.725294 9009 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
04:01:25.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:25.725625 9009 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
04:01:25.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:25.725831 9009 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
04:01:25.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:25.725977 9009 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40281/jwks.json
04:01:25.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:25.763242 9009 master_runner.cc:386] Master server non-default flags:
04:01:25.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:01:25.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.112.212:35587
04:01:25.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:01:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:01:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-0/data
04:01:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-0/wal
04:01:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
04:01:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:01:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.112.254:41699,127.8.112.253:44153,127.8.112.252:42515
04:01:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/krb5kdc/kudu_127.8.112.254.keytab
04:01:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.112.254
04:01:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:01:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:01:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:01:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.112.254:41699
04:01:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:01:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
04:01:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:01:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40281/jwks.json
04:01:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
04:01:25.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
04:01:25.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
04:01:25.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:01:25.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:01:25.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-0/data/info.pb
04:01:25.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.112.254
04:01:25.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43437
04:01:25.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
04:01:25.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:01:25.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
04:01:25.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
04:01:25.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:01:25.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-0/logs
04:01:25.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:01:25.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
04:01:25.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
04:01:25.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
04:01:25.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
04:01:25.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
04:01:25.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
04:01:25.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 02:58:30 UTC on ccycloud.quasar-xnspdp.root.comops.site
04:01:25.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4569537
04:01:25.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
04:01:25.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:25.764818 9009 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
04:01:25.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:25.766186 9009 file_cache.cc:493] Constructed file cache file cache with capacity 419430
04:01:25.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:25.783794 9016 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:01:27.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:25.783794 9015 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:01:27.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:25.786347 9009 server_base.cc:1034] running on GCE node
04:01:27.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:25.784440 9018 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:01:27.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:27.008236 9009 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
04:01:27.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:27.011076 9009 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
04:01:27.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:27.012421 9009 hybrid_clock.cc:648] HybridClock initialized: now 1785988887012396 us; error 64 us; skew 500 ppm
04:01:27.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:27.013223 9009 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
04:01:27.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 04:01:27 dist-test-slave-1jjb krb5kdc[8656](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1785988887, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.112.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
04:01:27.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:27.024446 9009 init.cc:377] Logged in from keytab as kudu/127.8.112.254@KRBTEST.COM (short username kudu)
04:01:27.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:27.030478 9009 webserver.cc:460] Webserver started at http://127.8.112.254:43437/ using document root <none> and password file <none>
04:01:27.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:27.031558 9009 fs_manager.cc:359] Metadata directory not provided
04:01:27.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:27.031764 9009 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
04:01:27.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:27.040822 9009 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.007s sys 0.000s
04:01:27.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:27.045429 9028 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
04:01:27.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:27.046758 9009 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
04:01:27.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:27.047176 9009 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-0/data,/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-0/wal
04:01:27.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7e5cffbbe0ff4b0ca459bb5fd221108b"
04:01:27.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 04:01:19 on dist-test-slave-1jjb"
04:01:27.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:27.050364 9009 fs_report.cc:383] FS layout report
04:01:27.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
04:01:27.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-0/wal
04:01:27.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-0/wal
04:01:27.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-0/data/data
04:01:27.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
04:01:27.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
04:01:27.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
04:01:27.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
04:01:27.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
04:01:27.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
04:01:27.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
04:01:27.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
04:01:27.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
04:01:27.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
04:01:27.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
04:01:27.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:27.088897 9009 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
04:01:27.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:27.095769 9009 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
04:01:27.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:27.096352 9009 kserver.cc:163] Server-wide thread pool size limit: 3276
04:01:27.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:27.170038 9009 rpc_server.cc:307] RPC server started. Bound to: 127.8.112.254:41699
04:01:27.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:27.170212 9079 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.8.112.254:41699 every 8 connection(s)
04:01:27.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:27.173465 9009 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-0/data/info.pb
04:01:27.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:27.181644 8643 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskKPlGYQ/build/tsan/bin/kudu as pid 9009
04:01:27.183 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
04:01:27.183 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.8.112.252:42515
04:01:27.183 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
04:01:27.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:27.184719 8643 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskKPlGYQ/build/tsan/bin/kudu
04:01:27.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskKPlGYQ/build/tsan/bin/kudu
04:01:27.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-2/wal
04:01:27.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-2/data
04:01:27.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
04:01:27.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
04:01:27.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
04:01:27.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:01:27.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:01:27.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:01:27.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-2/logs
04:01:27.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-2/data/info.pb
04:01:27.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:01:27.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
04:01:27.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
04:01:27.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
04:01:27.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
04:01:27.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:01:27.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:01:27.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:01:27.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
04:01:27.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
04:01:27.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:01:27.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:01:27.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.112.252:42515
04:01:27.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.112.252
04:01:27.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46415
04:01:27.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/krb5kdc/kudu_127.8.112.252.keytab
04:01:27.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.112.252
04:01:27.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
04:01:27.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
04:01:27.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
04:01:27.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.112.212:35587
04:01:27.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:01:27.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:01:27.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:01:27.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:01:27.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.112.254:41699,127.8.112.253:44153,127.8.112.252:42515
04:01:27.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:01:27.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40281/jwks.json
04:01:27.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
04:01:27.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:27.184823 9080 sys_catalog.cc:263] Verifying existing consensus state
04:01:27.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260806 04:01:27.185916 9080 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
04:01:27.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:27.186441 9009 master.cc:537] Master@127.8.112.254:41699 shutting down...
04:01:27.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:27.251731 9009 master.cc:552] Master@127.8.112.254:41699 shutdown complete.
04:01:27.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
04:01:27.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:27.530565 9082 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
04:01:27.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:27.531232 9082 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
04:01:27.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:27.531840 9082 flags.cc:425] Enabled unsafe flag: --never_fsync=true
04:01:27.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:27.567221 9082 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
04:01:27.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:27.567456 9082 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
04:01:27.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:27.567727 9082 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
04:01:27.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:27.567925 9082 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
04:01:27.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:27.568188 9082 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
04:01:27.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:27.568401 9082 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
04:01:27.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:27.568569 9082 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40281/jwks.json
04:01:27.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:27.609882 9082 master_runner.cc:386] Master server non-default flags:
04:01:27.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:01:27.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.112.212:35587
04:01:27.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:01:27.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:01:27.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-2/data
04:01:27.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-2/wal
04:01:27.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
04:01:27.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:01:27.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.112.254:41699,127.8.112.253:44153,127.8.112.252:42515
04:01:27.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/krb5kdc/kudu_127.8.112.252.keytab
04:01:27.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.112.252
04:01:27.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:01:27.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:01:27.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:01:27.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.112.252:42515
04:01:27.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:01:27.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
04:01:27.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:01:27.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40281/jwks.json
04:01:27.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
04:01:27.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
04:01:27.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
04:01:27.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:01:27.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:01:27.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-2/data/info.pb
04:01:27.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.112.252
04:01:27.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46415
04:01:27.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
04:01:27.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:01:27.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
04:01:27.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
04:01:27.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:01:27.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-2/logs
04:01:27.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:01:27.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
04:01:27.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
04:01:27.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
04:01:27.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
04:01:27.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
04:01:27.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
04:01:27.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 02:58:30 UTC on ccycloud.quasar-xnspdp.root.comops.site
04:01:27.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4569537
04:01:27.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
04:01:27.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:27.611194 9082 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
04:01:27.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:27.612304 9082 file_cache.cc:493] Constructed file cache file cache with capacity 419430
04:01:27.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:27.629733 9088 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:01:28.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:27.632709 9082 server_base.cc:1034] running on GCE node
04:01:28.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:27.629994 9089 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:01:28.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:27.631423 9091 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:01:28.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:28.872660 9082 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
04:01:28.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:28.875182 9082 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
04:01:28.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:28.876530 9082 hybrid_clock.cc:648] HybridClock initialized: now 1785988888876502 us; error 49 us; skew 500 ppm
04:01:28.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:28.877282 9082 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
04:01:28.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 04:01:28 dist-test-slave-1jjb krb5kdc[8656](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1785988888, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.112.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
04:01:28.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:28.887230 9082 init.cc:377] Logged in from keytab as kudu/127.8.112.252@KRBTEST.COM (short username kudu)
04:01:28.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:28.892795 9082 webserver.cc:460] Webserver started at http://127.8.112.252:46415/ using document root <none> and password file <none>
04:01:28.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:28.893860 9082 fs_manager.cc:359] Metadata directory not provided
04:01:28.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:28.894131 9082 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
04:01:28.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:28.903122 9082 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.000s sys 0.005s
04:01:28.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:28.907630 9099 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
04:01:28.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:28.908800 9082 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
04:01:28.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:28.909202 9082 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-2/data,/tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-2/wal
04:01:28.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0c56e39950cc4685bfaff0cfc2d7b48d"
04:01:28.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 04:01:22 on dist-test-slave-1jjb"
04:01:28.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:28.911731 9082 fs_report.cc:383] FS layout report
04:01:28.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
04:01:28.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-2/wal
04:01:28.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-2/wal
04:01:28.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-2/data/data
04:01:28.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
04:01:28.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
04:01:28.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
04:01:28.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
04:01:28.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
04:01:28.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
04:01:28.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
04:01:28.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
04:01:28.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
04:01:28.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
04:01:28.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
04:01:28.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:28.957703 9082 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
04:01:28.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:28.963634 9082 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
04:01:28.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:28.964227 9082 kserver.cc:163] Server-wide thread pool size limit: 3276
04:01:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:29.045989 9082 rpc_server.cc:307] RPC server started. Bound to: 127.8.112.252:42515
04:01:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:29.046123 9150 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.8.112.252:42515 every 8 connection(s)
04:01:29.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:29.050249 9082 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskKPlGYQ/test-tmp/mini-kudu-cluster8390760955931249743/master-2/data/info.pb
04:01:29.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:29.057577 8643 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskKPlGYQ/build/tsan/bin/kudu as pid 9082
04:01:29.058 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
04:01:29.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:29.063715 9151 sys_catalog.cc:263] Verifying existing consensus state
04:01:29.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:29.071760 9151 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d: Bootstrap starting.
04:01:29.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:29.112653 9151 log.cc:824] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d: Log is configured to *not* fsync() on all Append() calls
04:01:29.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:29.117460 9151 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
04:01:29.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:29.118218 9151 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d: Bootstrap complete.
04:01:29.118 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.8.112.253:44153(127.8.112.253:44153)] connecting to peer
04:01:29.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:29.143265 9151 raft_consensus.cc:348] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7e5cffbbe0ff4b0ca459bb5fd221108b" member_type: VOTER last_known_addr { host: "127.8.112.254" port: 41699 } } peers { permanent_uuid: "d556530449f84860bc4c8edc9e606fd9" member_type: VOTER last_known_addr { host: "127.8.112.253" port: 44153 } } peers { permanent_uuid: "0c56e39950cc4685bfaff0cfc2d7b48d" member_type: VOTER last_known_addr { host: "127.8.112.252" port: 42515 } }
04:01:29.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:29.146092 9151 raft_consensus.cc:374] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
04:01:29.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:29.146414 9151 raft_consensus.cc:729] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0c56e39950cc4685bfaff0cfc2d7b48d, State: Initialized, Role: FOLLOWER
04:01:29.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:29.147537 9151 consensus_queue.cc:260] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7e5cffbbe0ff4b0ca459bb5fd221108b" member_type: VOTER last_known_addr { host: "127.8.112.254" port: 41699 } } peers { permanent_uuid: "d556530449f84860bc4c8edc9e606fd9" member_type: VOTER last_known_addr { host: "127.8.112.253" port: 44153 } } peers { permanent_uuid: "0c56e39950cc4685bfaff0cfc2d7b48d" member_type: VOTER last_known_addr { host: "127.8.112.252" port: 42515 } }
04:01:29.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:29.151063 9155 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7e5cffbbe0ff4b0ca459bb5fd221108b" member_type: VOTER last_known_addr { host: "127.8.112.254" port: 41699 } } peers { permanent_uuid: "d556530449f84860bc4c8edc9e606fd9" member_type: VOTER last_known_addr { host: "127.8.112.253" port: 44153 } } peers { permanent_uuid: "0c56e39950cc4685bfaff0cfc2d7b48d" member_type: VOTER last_known_addr { host: "127.8.112.252" port: 42515 } } }
04:01:29.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:29.152293 9155 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d [sys.catalog]: This master's current role is: FOLLOWER
04:01:29.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:29.153095 9151 sys_catalog.cc:564] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d [sys.catalog]: configured and running, proceeding with master startup.
04:01:29.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:29.187925 9167 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:01:29.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:29.188292 9167 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:01:29.193 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.8.112.254:41699(127.8.112.254:41699)] connecting to peer
04:01:29.204 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.8.112.252:42515(127.8.112.252:42515)] connecting to peer
04:01:29.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:29.185745 9082 master_runner.cc:186] Error getting master registration for 127.8.112.254:41699: Network error: Client connection negotiation failed: client connection to 127.8.112.254:41699: connect: Connection refused (error 111),
04:01:29.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:29.218510 9082 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
04:01:29.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:29.219733 9105 tablet.cc:2369] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d: Can't schedule compaction. Clean time has not been advanced past its initial value.
04:01:29.238 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
04:01:29.239 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.112.254:41699(127.8.112.254:41699)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
04:01:29.240 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153
04:01:29.241 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.112.253:44153(127.8.112.253:44153)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153
04:01:29.243 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.112.253:44153: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153
04:01:29.244 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.8.112.252:42515(127.8.112.252:42515)] Successfully connected to peer
04:01:29.246 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.112.254:41699: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
04:01:29.336 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
04:01:29.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:29.339607 9155 raft_consensus.cc:482] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
04:01:29.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:29.340206 9155 raft_consensus.cc:504] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7e5cffbbe0ff4b0ca459bb5fd221108b" member_type: VOTER last_known_addr { host: "127.8.112.254" port: 41699 } } peers { permanent_uuid: "d556530449f84860bc4c8edc9e606fd9" member_type: VOTER last_known_addr { host: "127.8.112.253" port: 44153 } } peers { permanent_uuid: "0c56e39950cc4685bfaff0cfc2d7b48d" member_type: VOTER last_known_addr { host: "127.8.112.252" port: 42515 } }
04:01:29.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:29.345077 9103 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.112.254:41699: connect: Connection refused (error 111)
04:01:29.348 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.8.112.252
04:01:29.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:29.351142 9155 leader_election.cc:288] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7e5cffbbe0ff4b0ca459bb5fd221108b (127.8.112.254:41699), d556530449f84860bc4c8edc9e606fd9 (127.8.112.253:44153)
04:01:29.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:29.360659 9102 leader_election.cc:334] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d556530449f84860bc4c8edc9e606fd9 (127.8.112.253:44153): Network error: Client connection negotiation failed: client connection to 127.8.112.253:44153: connect: Connection refused (error 111)
04:01:29.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:29.363852 9103 leader_election.cc:334] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 7e5cffbbe0ff4b0ca459bb5fd221108b (127.8.112.254:41699): Network error: Client connection negotiation failed: client connection to 127.8.112.254:41699: connect: Connection refused (error 111)
04:01:29.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:29.364336 9103 leader_election.cc:302] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 0c56e39950cc4685bfaff0cfc2d7b48d; no voters: 7e5cffbbe0ff4b0ca459bb5fd221108b, d556530449f84860bc4c8edc9e606fd9
04:01:29.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:29.365450 9155 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
04:01:29.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:29.577601 8145 debug-util.cc:398] Leaking SignalData structure 0x7b080009c1c0 after lost signal to thread 8075
04:01:29.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:29.578425 8145 debug-util.cc:398] Leaking SignalData structure 0x7b080009f460 after lost signal to thread 8148
04:01:29.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 04:01:29 dist-test-slave-1jjb krb5kdc[8656](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1785988883, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.8.112.252@KRBTEST.COM
04:01:29.640 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xa7920fe5, L:/127.0.0.1:47338 - R:/127.8.112.252:42515] using SASL/GSSAPI
04:01:29.663 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515; will retry
04:01:29.686 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.112.253:44153(127.8.112.253:44153)] connecting to peer
04:01:29.688 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.112.254:41699(127.8.112.254:41699)] connecting to peer
04:01:29.698 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153
04:01:29.698 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.112.253:44153(127.8.112.253:44153)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153
04:01:29.698 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.112.253:44153: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153
04:01:29.701 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
04:01:29.702 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.112.254:41699(127.8.112.254:41699)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
04:01:29.702 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.112.254:41699: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
04:01:29.705 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515; will retry
04:01:29.725 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.112.253:44153(127.8.112.253:44153)] connecting to peer
04:01:29.726 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.112.254:41699(127.8.112.254:41699)] connecting to peer
04:01:29.728 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153
04:01:29.728 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.112.253:44153(127.8.112.253:44153)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153
04:01:29.729 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.112.253:44153: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153
04:01:29.735 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
04:01:29.735 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.112.254:41699(127.8.112.254:41699)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
04:01:29.736 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.112.254:41699: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
04:01:29.737 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515; will retry
04:01:29.765 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.112.253:44153(127.8.112.253:44153)] connecting to peer
04:01:29.767 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.112.254:41699(127.8.112.254:41699)] connecting to peer
04:01:29.768 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153
04:01:29.768 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.112.253:44153(127.8.112.253:44153)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153
04:01:29.769 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.112.253:44153: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153
04:01:29.773 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
04:01:29.773 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.112.254:41699(127.8.112.254:41699)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
04:01:29.774 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.112.254:41699: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
04:01:29.777 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515; will retry
04:01:29.805 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.112.253:44153(127.8.112.253:44153)] connecting to peer
04:01:29.807 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.112.254:41699(127.8.112.254:41699)] connecting to peer
04:01:29.807 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153
04:01:29.807 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.112.253:44153(127.8.112.253:44153)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153
04:01:29.809 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.112.253:44153: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153
04:01:29.811 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
04:01:29.811 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.112.254:41699(127.8.112.254:41699)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
04:01:29.812 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.112.254:41699: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
04:01:29.815 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515; will retry
04:01:29.845 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.112.253:44153(127.8.112.253:44153)] connecting to peer
04:01:29.846 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.112.254:41699(127.8.112.254:41699)] connecting to peer
04:01:29.847 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153
04:01:29.847 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.112.253:44153(127.8.112.253:44153)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153
04:01:29.847 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.112.253:44153: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153
04:01:29.860 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
04:01:29.861 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.112.254:41699(127.8.112.254:41699)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
04:01:29.861 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.112.254:41699: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
04:01:29.862 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515; will retry
04:01:29.925 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.112.253:44153(127.8.112.253:44153)] connecting to peer
04:01:29.927 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.112.254:41699(127.8.112.254:41699)] connecting to peer
04:01:29.927 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153
04:01:29.928 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.112.253:44153(127.8.112.253:44153)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153
04:01:29.928 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.112.253:44153: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153
04:01:29.932 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
04:01:29.932 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.112.254:41699(127.8.112.254:41699)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
04:01:29.933 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.112.254:41699: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
04:01:29.936 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515; will retry
04:01:30.065 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.112.253:44153(127.8.112.253:44153)] connecting to peer
04:01:30.067 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.112.254:41699(127.8.112.254:41699)] connecting to peer
04:01:30.075 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153
04:01:30.075 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.112.253:44153(127.8.112.253:44153)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153
04:01:30.077 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
04:01:30.077 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.112.254:41699(127.8.112.254:41699)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
04:01:30.078 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.112.253:44153: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153
04:01:30.078 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.112.254:41699: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
04:01:30.080 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515; will retry
04:01:30.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:30.192970 9167 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:01:30.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:30.193651 9167 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:01:30.265 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.112.253:44153(127.8.112.253:44153)] connecting to peer
04:01:30.268 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153
04:01:30.268 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.112.254:41699(127.8.112.254:41699)] connecting to peer
04:01:30.268 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.112.253:44153(127.8.112.253:44153)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153
04:01:30.270 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.112.253:44153: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153
04:01:30.271 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
04:01:30.271 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.112.254:41699(127.8.112.254:41699)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
04:01:30.271 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.112.254:41699: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
04:01:30.276 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515; will retry
04:01:30.485 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.112.253:44153(127.8.112.253:44153)] connecting to peer
04:01:30.488 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.112.254:41699(127.8.112.254:41699)] connecting to peer
04:01:30.489 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153
04:01:30.489 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.112.253:44153(127.8.112.253:44153)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153
04:01:30.490 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.112.253:44153: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153
04:01:30.491 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
04:01:30.491 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.112.254:41699(127.8.112.254:41699)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
04:01:30.492 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.112.254:41699: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
04:01:30.498 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515; will retry
04:01:31.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:31.198554 9167 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:01:31.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:31.198907 9167 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:01:31.265 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.112.253:44153(127.8.112.253:44153)] connecting to peer
04:01:31.267 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.112.254:41699(127.8.112.254:41699)] connecting to peer
04:01:31.268 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153
04:01:31.268 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.112.253:44153(127.8.112.253:44153)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153
04:01:31.268 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.112.253:44153: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153
04:01:31.271 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
04:01:31.272 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.112.254:41699(127.8.112.254:41699)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
04:01:31.272 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.112.254:41699: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
04:01:31.274 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515; will retry
04:01:31.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:31.608516 9176 raft_consensus.cc:482] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
04:01:31.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:31.608920 9176 raft_consensus.cc:504] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7e5cffbbe0ff4b0ca459bb5fd221108b" member_type: VOTER last_known_addr { host: "127.8.112.254" port: 41699 } } peers { permanent_uuid: "d556530449f84860bc4c8edc9e606fd9" member_type: VOTER last_known_addr { host: "127.8.112.253" port: 44153 } } peers { permanent_uuid: "0c56e39950cc4685bfaff0cfc2d7b48d" member_type: VOTER last_known_addr { host: "127.8.112.252" port: 42515 } }
04:01:31.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:31.610371 9176 leader_election.cc:288] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7e5cffbbe0ff4b0ca459bb5fd221108b (127.8.112.254:41699), d556530449f84860bc4c8edc9e606fd9 (127.8.112.253:44153)
04:01:31.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:31.616746 9102 leader_election.cc:334] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d556530449f84860bc4c8edc9e606fd9 (127.8.112.253:44153): Network error: Client connection negotiation failed: client connection to 127.8.112.253:44153: connect: Connection refused (error 111)
04:01:31.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:31.617193 9103 leader_election.cc:334] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 7e5cffbbe0ff4b0ca459bb5fd221108b (127.8.112.254:41699): Network error: Client connection negotiation failed: client connection to 127.8.112.254:41699: connect: Connection refused (error 111)
04:01:31.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:31.617564 9103 leader_election.cc:302] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 0c56e39950cc4685bfaff0cfc2d7b48d; no voters: 7e5cffbbe0ff4b0ca459bb5fd221108b, d556530449f84860bc4c8edc9e606fd9
04:01:31.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:31.618223 9176 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
04:01:32.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:32.201960 9167 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:01:32.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:32.202209 9167 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:01:32.625 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.112.253:44153(127.8.112.253:44153)] connecting to peer
04:01:32.626 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.112.254:41699(127.8.112.254:41699)] connecting to peer
04:01:32.627 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153
04:01:32.628 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.112.253:44153(127.8.112.253:44153)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153
04:01:32.628 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.112.253:44153: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153
04:01:32.629 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
04:01:32.629 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.112.254:41699(127.8.112.254:41699)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
04:01:32.629 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.112.254:41699: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
04:01:32.632 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515; will retry
04:01:33.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:33.206580 9167 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:01:33.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:33.209082 9167 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:01:34.145 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.112.253:44153(127.8.112.253:44153)] connecting to peer
04:01:34.146 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.112.254:41699(127.8.112.254:41699)] connecting to peer
04:01:34.147 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153
04:01:34.147 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.112.253:44153(127.8.112.253:44153)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153
04:01:34.149 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
04:01:34.149 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.112.254:41699(127.8.112.254:41699)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
04:01:34.149 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.112.254:41699: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
04:01:34.150 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.112.253:44153: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153
04:01:34.152 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515; will retry
04:01:34.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:34.212226 9167 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:01:34.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:34.212468 9167 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:01:34.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:34.221149 9082 master_runner.cc:186] Error getting master registration for 127.8.112.254:41699: Network error: Client connection negotiation failed: client connection to 127.8.112.254:41699: connect: Connection refused (error 111),
04:01:34.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:34.221515 9082 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
04:01:34.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:34.234241 8487 debug-util.cc:398] Leaking SignalData structure 0x7b08000afca0 after lost signal to thread 8361
04:01:34.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:34.235328 8487 debug-util.cc:398] Leaking SignalData structure 0x7b08000af8c0 after lost signal to thread 8490
04:01:34.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:34.590703 9186 raft_consensus.cc:482] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
04:01:34.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:34.591135 9186 raft_consensus.cc:504] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7e5cffbbe0ff4b0ca459bb5fd221108b" member_type: VOTER last_known_addr { host: "127.8.112.254" port: 41699 } } peers { permanent_uuid: "d556530449f84860bc4c8edc9e606fd9" member_type: VOTER last_known_addr { host: "127.8.112.253" port: 44153 } } peers { permanent_uuid: "0c56e39950cc4685bfaff0cfc2d7b48d" member_type: VOTER last_known_addr { host: "127.8.112.252" port: 42515 } }
04:01:34.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:34.592962 9186 leader_election.cc:288] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7e5cffbbe0ff4b0ca459bb5fd221108b (127.8.112.254:41699), d556530449f84860bc4c8edc9e606fd9 (127.8.112.253:44153)
04:01:34.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:34.594611 9103 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.112.254:41699: connect: Connection refused (error 111) [suppressed 3 similar messages]
04:01:34.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:34.598690 9103 leader_election.cc:334] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 7e5cffbbe0ff4b0ca459bb5fd221108b (127.8.112.254:41699): Network error: Client connection negotiation failed: client connection to 127.8.112.254:41699: connect: Connection refused (error 111)
04:01:34.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:34.599406 9102 leader_election.cc:334] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d556530449f84860bc4c8edc9e606fd9 (127.8.112.253:44153): Network error: Client connection negotiation failed: client connection to 127.8.112.253:44153: connect: Connection refused (error 111)
04:01:34.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:34.599867 9102 leader_election.cc:302] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 0c56e39950cc4685bfaff0cfc2d7b48d; no voters: 7e5cffbbe0ff4b0ca459bb5fd221108b, d556530449f84860bc4c8edc9e606fd9
04:01:34.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:34.600705 9186 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
04:01:35.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:35.216876 9167 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:01:35.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:35.217255 9167 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:01:35.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:35.903312 8628 debug-util.cc:398] Leaking SignalData structure 0x7b08000bcc60 after lost signal to thread 8502
04:01:35.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:35.904584 8628 debug-util.cc:398] Leaking SignalData structure 0x7b08000bcbc0 after lost signal to thread 8631
04:01:36.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:36.165764 9190 raft_consensus.cc:482] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
04:01:36.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:36.166204 9190 raft_consensus.cc:504] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7e5cffbbe0ff4b0ca459bb5fd221108b" member_type: VOTER last_known_addr { host: "127.8.112.254" port: 41699 } } peers { permanent_uuid: "d556530449f84860bc4c8edc9e606fd9" member_type: VOTER last_known_addr { host: "127.8.112.253" port: 44153 } } peers { permanent_uuid: "0c56e39950cc4685bfaff0cfc2d7b48d" member_type: VOTER last_known_addr { host: "127.8.112.252" port: 42515 } }
04:01:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:36.167829 9190 leader_election.cc:288] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7e5cffbbe0ff4b0ca459bb5fd221108b (127.8.112.254:41699), d556530449f84860bc4c8edc9e606fd9 (127.8.112.253:44153)
04:01:36.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:36.173364 9103 leader_election.cc:334] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 7e5cffbbe0ff4b0ca459bb5fd221108b (127.8.112.254:41699): Network error: Client connection negotiation failed: client connection to 127.8.112.254:41699: connect: Connection refused (error 111)
04:01:36.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:36.174326 9102 leader_election.cc:334] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d556530449f84860bc4c8edc9e606fd9 (127.8.112.253:44153): Network error: Client connection negotiation failed: client connection to 127.8.112.253:44153: connect: Connection refused (error 111)
04:01:36.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:36.174768 9102 leader_election.cc:302] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 0c56e39950cc4685bfaff0cfc2d7b48d; no voters: 7e5cffbbe0ff4b0ca459bb5fd221108b, d556530449f84860bc4c8edc9e606fd9
04:01:36.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:36.175506 9190 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
04:01:36.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:36.220268 9167 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:01:36.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:36.220544 9167 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:01:37.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:37.223680 9167 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:01:37.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:37.223976 9167 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:01:38.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:38.227999 9167 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:01:38.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:38.228471 9167 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:01:38.245 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.112.253:44153(127.8.112.253:44153)] connecting to peer
04:01:38.246 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.112.254:41699(127.8.112.254:41699)] connecting to peer
04:01:38.247 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153
04:01:38.247 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.112.253:44153(127.8.112.253:44153)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153
04:01:38.248 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.112.253:44153: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153
04:01:38.249 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
04:01:38.249 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.112.254:41699(127.8.112.254:41699)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
04:01:38.250 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.112.254:41699: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
04:01:38.252 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515; will retry
04:01:38.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 04:01:38 dist-test-slave-1jjb krb5kdc[8656](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1785988898, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.112.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
04:01:38.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:38.891162 9095 init.cc:295] Successfully reacquired a new kerberos TGT
04:01:39.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:39.224845 9082 master_runner.cc:186] Error getting master registration for 127.8.112.254:41699: Network error: Client connection negotiation failed: client connection to 127.8.112.254:41699: connect: Connection refused (error 111),
04:01:39.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:39.225260 9082 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
04:01:39.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:39.231544 9167 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:01:39.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:39.231807 9167 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:01:39.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:39.914011 9199 raft_consensus.cc:482] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
04:01:39.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:39.914492 9199 raft_consensus.cc:504] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7e5cffbbe0ff4b0ca459bb5fd221108b" member_type: VOTER last_known_addr { host: "127.8.112.254" port: 41699 } } peers { permanent_uuid: "d556530449f84860bc4c8edc9e606fd9" member_type: VOTER last_known_addr { host: "127.8.112.253" port: 44153 } } peers { permanent_uuid: "0c56e39950cc4685bfaff0cfc2d7b48d" member_type: VOTER last_known_addr { host: "127.8.112.252" port: 42515 } }
04:01:39.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:39.916067 9199 leader_election.cc:288] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7e5cffbbe0ff4b0ca459bb5fd221108b (127.8.112.254:41699), d556530449f84860bc4c8edc9e606fd9 (127.8.112.253:44153)
04:01:39.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:39.917996 9103 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.112.254:41699: connect: Connection refused (error 111) [suppressed 3 similar messages]
04:01:39.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:39.920797 9103 leader_election.cc:334] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 7e5cffbbe0ff4b0ca459bb5fd221108b (127.8.112.254:41699): Network error: Client connection negotiation failed: client connection to 127.8.112.254:41699: connect: Connection refused (error 111)
04:01:39.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:39.921299 9102 leader_election.cc:334] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d556530449f84860bc4c8edc9e606fd9 (127.8.112.253:44153): Network error: Client connection negotiation failed: client connection to 127.8.112.253:44153: connect: Connection refused (error 111)
04:01:39.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:39.921572 9102 leader_election.cc:302] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 0c56e39950cc4685bfaff0cfc2d7b48d; no voters: 7e5cffbbe0ff4b0ca459bb5fd221108b, d556530449f84860bc4c8edc9e606fd9
04:01:39.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:39.922122 9199 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
04:01:40.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:40.234995 9167 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:01:40.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:40.235296 9167 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:01:41.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:41.238179 9167 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:01:41.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:41.238440 9167 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:01:41.746 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.112.253:44153(127.8.112.253:44153)] connecting to peer
04:01:41.747 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.112.254:41699(127.8.112.254:41699)] connecting to peer
04:01:41.748 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153
04:01:41.749 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.112.253:44153(127.8.112.253:44153)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153
04:01:41.749 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.112.253:44153: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153
04:01:41.750 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
04:01:41.750 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.112.254:41699(127.8.112.254:41699)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
04:01:41.750 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.112.254:41699: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
04:01:41.753 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515; will retry
04:01:42.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:42.241266 9167 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:01:42.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:42.241583 9167 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:01:43.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:43.244606 9167 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:01:43.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:43.244952 9167 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:01:44.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:44.227965 9082 master_runner.cc:186] Error getting master registration for 127.8.112.254:41699: Network error: Client connection negotiation failed: client connection to 127.8.112.254:41699: connect: Connection refused (error 111),
04:01:44.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:44.228387 9082 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
04:01:44.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:44.247983 9167 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:01:44.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:44.248377 9167 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:01:44.265 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.112.253:44153(127.8.112.253:44153)] connecting to peer
04:01:44.266 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.112.254:41699(127.8.112.254:41699)] connecting to peer
04:01:44.268 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153
04:01:44.268 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.112.253:44153(127.8.112.253:44153)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153
04:01:44.268 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.112.253:44153: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153
04:01:44.269 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
04:01:44.269 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.112.254:41699(127.8.112.254:41699)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
04:01:44.269 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.112.254:41699: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
04:01:44.271 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515; will retry
04:01:45.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:45.251645 9167 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:01:45.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:45.251921 9167 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:01:46.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:46.027448 8145 debug-util.cc:398] Leaking SignalData structure 0x7b08000c1cc0 after lost signal to thread 8075
04:01:46.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:46.028162 8145 debug-util.cc:398] Leaking SignalData structure 0x7b08000b7120 after lost signal to thread 8148
04:01:46.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:46.255098 9167 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:01:46.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:46.255386 9167 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:01:46.725 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.112.253:44153(127.8.112.253:44153)] connecting to peer
04:01:46.727 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.112.254:41699(127.8.112.254:41699)] connecting to peer
04:01:46.728 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153
04:01:46.728 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.112.253:44153(127.8.112.253:44153)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153
04:01:46.728 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.112.253:44153: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153
04:01:46.729 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
04:01:46.729 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.112.254:41699(127.8.112.254:41699)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
04:01:46.730 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.112.254:41699: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
04:01:46.732 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515; will retry
04:01:47.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:47.258365 9167 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:01:47.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:47.258665 9167 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:01:47.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:47.791241 7988 debug-util.cc:398] Leaking SignalData structure 0x7b08000a4ee0 after lost signal to thread 7924
04:01:47.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:47.791940 7988 debug-util.cc:398] Leaking SignalData structure 0x7b08000b1b00 after lost signal to thread 7991
04:01:48.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:48.262269 9167 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:01:48.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:48.262601 9167 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:01:48.465 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.112.253:44153(127.8.112.253:44153)] connecting to peer
04:01:48.466 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.112.254:41699(127.8.112.254:41699)] connecting to peer
04:01:48.469 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153
04:01:48.469 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.112.253:44153(127.8.112.253:44153)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153
04:01:48.469 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.112.253:44153: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153
04:01:48.478 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
04:01:48.478 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.112.254:41699(127.8.112.254:41699)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
04:01:48.478 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.112.254:41699: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
04:01:48.481 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515; will retry
04:01:48.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 04:01:48 dist-test-slave-1jjb krb5kdc[8656](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1785988908, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.112.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
04:01:48.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:48.893671 9095 init.cc:295] Successfully reacquired a new kerberos TGT
04:01:49.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:49.232414 9082 master_runner.cc:186] Error getting master registration for 127.8.112.254:41699: Network error: Client connection negotiation failed: client connection to 127.8.112.254:41699: connect: Connection refused (error 111),
04:01:49.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:49.232872 9082 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
04:01:49.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:49.265599 9167 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:01:49.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:49.265908 9167 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:01:49.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:49.780424 9216 raft_consensus.cc:482] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
04:01:49.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:49.780855 9216 raft_consensus.cc:504] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7e5cffbbe0ff4b0ca459bb5fd221108b" member_type: VOTER last_known_addr { host: "127.8.112.254" port: 41699 } } peers { permanent_uuid: "d556530449f84860bc4c8edc9e606fd9" member_type: VOTER last_known_addr { host: "127.8.112.253" port: 44153 } } peers { permanent_uuid: "0c56e39950cc4685bfaff0cfc2d7b48d" member_type: VOTER last_known_addr { host: "127.8.112.252" port: 42515 } }
04:01:49.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:49.782567 9216 leader_election.cc:288] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7e5cffbbe0ff4b0ca459bb5fd221108b (127.8.112.254:41699), d556530449f84860bc4c8edc9e606fd9 (127.8.112.253:44153)
04:01:49.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:49.784921 9103 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.112.254:41699: connect: Connection refused (error 111) [suppressed 1 similar messages]
04:01:49.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:49.787628 9102 leader_election.cc:334] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d556530449f84860bc4c8edc9e606fd9 (127.8.112.253:44153): Network error: Client connection negotiation failed: client connection to 127.8.112.253:44153: connect: Connection refused (error 111)
04:01:49.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:49.788127 9103 leader_election.cc:334] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 7e5cffbbe0ff4b0ca459bb5fd221108b (127.8.112.254:41699): Network error: Client connection negotiation failed: client connection to 127.8.112.254:41699: connect: Connection refused (error 111)
04:01:49.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:49.788614 9103 leader_election.cc:302] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 0c56e39950cc4685bfaff0cfc2d7b48d; no voters: 7e5cffbbe0ff4b0ca459bb5fd221108b, d556530449f84860bc4c8edc9e606fd9
04:01:49.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:49.789398 9216 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
04:01:50.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:50.268846 9167 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:01:50.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:50.269124 9167 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:01:51.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:51.272109 9167 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:01:51.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:51.272356 9167 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:01:52.025 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.112.253:44153(127.8.112.253:44153)] connecting to peer
04:01:52.026 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.112.254:41699(127.8.112.254:41699)] connecting to peer
04:01:52.028 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153
04:01:52.028 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.112.253:44153(127.8.112.253:44153)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153
04:01:52.028 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.112.253:44153: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153
04:01:52.029 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
04:01:52.029 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.112.254:41699(127.8.112.254:41699)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
04:01:52.029 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.112.254:41699: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
04:01:52.031 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515; will retry
04:01:52.085 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.112.253:44153(127.8.112.253:44153)] connecting to peer
04:01:52.086 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.112.254:41699(127.8.112.254:41699)] connecting to peer
04:01:52.087 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153
04:01:52.088 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.112.253:44153(127.8.112.253:44153)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153
04:01:52.088 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.112.253:44153: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153
04:01:52.088 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
04:01:52.088 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.112.254:41699(127.8.112.254:41699)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
04:01:52.089 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.112.254:41699: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
04:01:52.091 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515; will retry
04:01:52.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:52.275650 9167 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:01:52.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:52.275926 9167 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:01:53.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:53.278877 9167 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:01:53.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:53.279173 9167 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:01:54.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:54.235771 9082 master_runner.cc:186] Error getting master registration for 127.8.112.254:41699: Network error: Client connection negotiation failed: client connection to 127.8.112.254:41699: connect: Connection refused (error 111),
04:01:54.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:54.236188 9082 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
04:01:54.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:54.282168 9167 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:01:54.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:54.282469 9167 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:01:54.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:54.693440 8346 debug-util.cc:398] Leaking SignalData structure 0x7b08000b8140 after lost signal to thread 8216
04:01:54.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:54.694285 8346 debug-util.cc:398] Leaking SignalData structure 0x7b08000afd20 after lost signal to thread 8349
04:01:55.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:55.285349 9167 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:01:55.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:55.285655 9167 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:01:55.765 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.112.253:44153(127.8.112.253:44153)] connecting to peer
04:01:55.766 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.112.254:41699(127.8.112.254:41699)] connecting to peer
04:01:55.766 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153
04:01:55.766 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.112.253:44153(127.8.112.253:44153)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153
04:01:55.766 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.112.253:44153: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153
04:01:55.767 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
04:01:55.767 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.112.254:41699(127.8.112.254:41699)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
04:01:55.768 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.112.254:41699: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
04:01:55.772 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515; will retry
04:01:56.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:56.288743 9167 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:01:56.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:56.289037 9167 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:01:57.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:57.291862 9167 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:01:57.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:57.292256 9167 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:01:58.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:58.296236 9167 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0c56e39950cc4685bfaff0cfc2d7b48d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:01:58.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 04:01:58.296573 9167 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:01:58.885 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.112.253:44153(127.8.112.253:44153)] connecting to peer
04:01:58.886 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.112.254:41699(127.8.112.254:41699)] connecting to peer
04:01:58.887 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153
04:01:58.887 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.112.253:44153(127.8.112.253:44153)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153
04:01:58.887 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.112.253:44153: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153
04:01:58.888 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
04:01:58.888 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.112.254:41699(127.8.112.254:41699)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
04:01:58.888 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.112.254:41699: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
04:01:58.891 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515; will retry
04:01:58.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 04:01:58 dist-test-slave-1jjb krb5kdc[8656](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1785988918, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.112.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
04:01:58.895 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=22, TimeoutTracker(timeout=30000, elapsed=29829), Traces: [0ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.253:44153, [125ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.254:41699, [136ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.252:42515, [173ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.253:44153: Network error: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153, [172ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.254:41699: Network error: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [590ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.252:42515: OK, [595ms] delaying RPC due to: Service unavailable: Master config (127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [617ms] refreshing cache from master, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.253:44153, [620ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.254:41699, [621ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.252:42515, [630ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.253:44153: Network error: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153, [634ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.254:41699: Network error: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [636ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.252:42515: OK, [637ms] delaying RPC due to: Service unavailable: Master config (127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [657ms] refreshing cache from master, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.253:44153, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.254:41699, [659ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.252:42515, [660ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.253:44153: Network error: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153, [666ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.252:42515: OK, [667ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.254:41699: Network error: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [669ms] delaying RPC due to: Service unavailable: Master config (127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [697ms] refreshing cache from master, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.253:44153, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.254:41699, [700ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.252:42515, [700ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.253:44153: Network error: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153, [706ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.254:41699: Network error: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [708ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.252:42515: OK, [711ms] delaying RPC due to: Service unavailable: Master config (127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [736ms] refreshing cache from master, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.253:44153, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.254:41699, [739ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.253:44153: Network error: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153, [740ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.252:42515, [743ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.254:41699: Network error: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [747ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.252:42515: OK, [748ms] delaying RPC due to: Service unavailable: Master config (127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [776ms] refreshing cache from master, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.253:44153, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.254:41699, [779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.252:42515, [779ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.253:44153: Network error: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153, [789ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.252:42515: OK, [793ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.254:41699: Network error: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [794ms] delaying RPC due to: Service unavailable: Master config (127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [856ms] refreshing cache from master, [857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.253:44153, [859ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.254:41699, [860ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.253:44153: Network error: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153, [860ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.252:42515, [865ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.254:41699: Network error: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [867ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.252:42515: OK, [868ms] delaying RPC due to: Service unavailable: Master config (127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [997ms] refreshing cache from master, [997ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.253:44153, [999ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.254:41699, [999ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.252:42515, [1006ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.252:42515: OK, [1007ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.253:44153: Network error: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153, [1009ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.254:41699: Network error: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [1012ms] delaying RPC due to: Service unavailable: Master config (127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [1196ms] refreshing cache from master, [1197ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.253:44153, [1200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.254:41699, [1200ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.253:44153: Network error: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153, [1202ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.252:42515, [1203ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.254:41699: Network error: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [1208ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.252:42515: OK, [1209ms] delaying RPC due to: Service unavailable: Master config (127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [1416ms] refreshing cache from master, [1417ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.253:44153, [1420ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.254:41699, [1421ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.253:44153: Network error: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153, [1423ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.254:41699: Network error: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [1425ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.252:42515, [1429ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.252:42515: OK, [1431ms] delaying RPC due to: Service unavailable: Master config (127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [2197ms] refreshing cache from master, [2197ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.253:44153, [2198ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.254:41699, [2200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.252:42515, [2200ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.253:44153: Network error: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153, [2204ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.254:41699: Network error: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [2206ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.252:42515: OK, [2206ms] delaying RPC due to: Service unavailable: Master config (127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [3556ms] refreshing cache from master, [3557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.253:44153, [3558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.254:41699, [3559ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.252:42515, [3560ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.253:44153: Network error: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153, [3561ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.254:41699: Network error: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [3563ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.252:42515: OK, [3564ms] delaying RPC due to: Service unavailable: Master config (127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [5076ms] refreshing cache from master, [5076ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.253:44153, [5077ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.254:41699, [5079ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.252:42515, [5081ms] too many traces: truncated at 100 traces, deferred=Deferred@151931680(state=PENDING, result=null, callback=(continuation of Deferred@611520581 after retry RPC after error@1419081759) -> (continuation of Deferred@952922256 after retry RPC after error@1138781154) -> (continuation of Deferred@2142087907 after retry RPC after error@483465) -> (continuation of Deferred@1528092905 after retry RPC after error@468391698) -> (continuation of Deferred@159603774 after retry RPC after error@851142319) -> (continuation of Deferred@1335178113 after retry RPC after error@210104038) -> (continuation of Deferred@527704548 after retry RPC after error@1989386939) -> (continuation of Deferred@276598483 after retry RPC after error@867016932) -> (continuation of Deferred@233364596 after retry RPC after error@1711612168) -> (continuation of Deferred@318451981 after retry RPC after error@1276053814) -> (continuation of Deferred@122644415 after retry RPC after error@1208818495) -> (continuation of Deferred@53815402 after retry RPC after error@935839619) -> (continuation of Deferred@1263086683 after retry RPC after error@1799160558) -> (continuation of Deferred@1096876443 after retry RPC after error@1217605099) -> (continuation of Deferred@1245379414 after retry RPC after error@1234418317) -> (continuation of Deferred@11995133 after retry RPC after error@1369083509) -> (continuation of Deferred@1281834843 after retry RPC after error@418689928) -> (continuation of Deferred@549771140 after retry RPC after error@623381711) -> (continuation of Deferred@1346396886 after retry RPC after error@1838543781) -> (continuation of Deferred@1802199396 after retry RPC after error@247486130) -> (continuation of Deferred@1573241710 after retry RPC after error@529468328), errback=(continuation of Deferred@611520581 after retry RPC after error@1419081759) -> (continuation of Deferred@952922256 after retry RPC after error@1138781154) -> (continuation of Deferred@2142087907 after retry RPC after error@483465) -> (continuation of Deferred@1528092905 after retry RPC after error@468391698) -> (continuation of Deferred@159603774 after retry RPC after error@851142319) -> (continuation of Deferred@1335178113 after retry RPC after error@210104038) -> (continuation of Deferred@527704548 after retry RPC after error@1989386939) -> (continuation of Deferred@276598483 after retry RPC after error@867016932) -> (continuation of Deferred@233364596 after retry RPC after error@1711612168) -> (continuation of Deferred@318451981 after retry RPC after error@1276053814) -> (continuation of Deferred@122644415 after retry RPC after error@1208818495) -> (continuation of Deferred@53815402 after retry RPC after error@935839619) -> (continuation of Deferred@1263086683 after retry RPC after error@1799160558) -> (continuation of Deferred@1096876443 after retry RPC after error@1217605099) -> (continuation of Deferred@1245379414 after retry RPC after error@1234418317) -> (continuation of Deferred@11995133 after retry RPC after error@1369083509) -> (continuation of Deferred@1281834843 after retry RPC after error@418689928) -> (continuation of Deferred@549771140 after retry RPC after error@623381711) -> (continuation of Deferred@1346396886 after retry RPC after error@1838543781) -> (continuation of Deferred@1802199396 after retry RPC after error@247486130) -> (continuation of Deferred@1573241710 after retry RPC after error@529468328)))
04:01:58.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:58.896579 9095 init.cc:295] Successfully reacquired a new kerberos TGT
04:01:58.900 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.8.112.252:42515(127.8.112.252:42515)] disconnecting while in state READY
04:01:58.901 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.8.112.254:41699(127.8.112.254:41699)] disconnecting while in state TERMINATED
04:01:58.901 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.8.112.253:44153(127.8.112.253:44153)] disconnecting while in state TERMINATED
04:01:58.905 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
04:01:58.907 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
04:01:58.909 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.8.112.252:42515(127.8.112.252:42515)] handling channelInactive
04:01:58.910 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.112.252:42515(127.8.112.252:42515)] cleaning up while in state READY due to: connection closed
04:01:58.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:01:58.911142 8643 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskKPlGYQ/build/tsan/bin/kudu with pid 9082
04:02:00.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-06T04:02:00Z chronyd exiting
04:02:01.179 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
04:02:01.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:02:01.180879 7911 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskKPlGYQ/build/tsan/bin/kudu with pid 8195
04:02:01.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:02:01.211874 7911 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskKPlGYQ/build/tsan/bin/kudu with pid 8360
04:02:01.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:02:01.244535 7911 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskKPlGYQ/build/tsan/bin/kudu with pid 8501
04:02:01.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:02:01.274628 7911 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskKPlGYQ/build/tsan/bin/kudu with pid 7923
04:02:01.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:02:01.315095 7911 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskKPlGYQ/build/tsan/bin/kudu with pid 7994
04:02:01.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 04:02:01.348927 7911 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskKPlGYQ/build/tsan/bin/kudu with pid 8069
04:02:02.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-06T04:02:02Z chronyd exiting
04:02:03.551 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=22, TimeoutTracker(timeout=30000, elapsed=29829), Traces: [0ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.253:44153, [125ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.254:41699, [136ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.252:42515, [173ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.253:44153: Network error: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153, [172ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.254:41699: Network error: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [590ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.252:42515: OK, [595ms] delaying RPC due to: Service unavailable: Master config (127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [617ms] refreshing cache from master, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.253:44153, [620ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.254:41699, [621ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.252:42515, [630ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.253:44153: Network error: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153, [634ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.254:41699: Network error: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [636ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.252:42515: OK, [637ms] delaying RPC due to: Service unavailable: Master config (127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [657ms] refreshing cache from master, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.253:44153, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.254:41699, [659ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.252:42515, [660ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.253:44153: Network error: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153, [666ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.252:42515: OK, [667ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.254:41699: Network error: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [669ms] delaying RPC due to: Service unavailable: Master config (127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [697ms] refreshing cache from master, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.253:44153, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.254:41699, [700ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.252:42515, [700ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.253:44153: Network error: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153, [706ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.254:41699: Network error: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [708ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.252:42515: OK, [711ms] delaying RPC due to: Service unavailable: Master config (127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [736ms] refreshing cache from master, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.253:44153, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.254:41699, [739ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.253:44153: Network error: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153, [740ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.252:42515, [743ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.254:41699: Network error: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [747ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.252:42515: OK, [748ms] delaying RPC due to: Service unavailable: Master config (127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [776ms] refreshing cache from master, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.253:44153, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.254:41699, [779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.252:42515, [779ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.253:44153: Network error: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153, [789ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.252:42515: OK, [793ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.254:41699: Network error: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [794ms] delaying RPC due to: Service unavailable: Master config (127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [856ms] refreshing cache from master, [857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.253:44153, [859ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.254:41699, [860ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.253:44153: Network error: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153, [860ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.252:42515, [865ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.254:41699: Network error: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [867ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.252:42515: OK, [868ms] delaying RPC due to: Service unavailable: Master config (127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [997ms] refreshing cache from master, [997ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.253:44153, [999ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.254:41699, [999ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.252:42515, [1006ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.252:42515: OK, [1007ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.253:44153: Network error: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153, [1009ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.254:41699: Network error: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [1012ms] delaying RPC due to: Service unavailable: Master config (127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [1196ms] refreshing cache from master, [1197ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.253:44153, [1200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.254:41699, [1200ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.253:44153: Network error: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153, [1202ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.252:42515, [1203ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.254:41699: Network error: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [1208ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.252:42515: OK, [1209ms] delaying RPC due to: Service unavailable: Master config (127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [1416ms] refreshing cache from master, [1417ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.253:44153, [1420ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.254:41699, [1421ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.253:44153: Network error: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153, [1423ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.254:41699: Network error: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [1425ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.252:42515, [1429ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.252:42515: OK, [1431ms] delaying RPC due to: Service unavailable: Master config (127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [2197ms] refreshing cache from master, [2197ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.253:44153, [2198ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.254:41699, [2200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.252:42515, [2200ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.253:44153: Network error: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153, [2204ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.254:41699: Network error: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [2206ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.252:42515: OK, [2206ms] delaying RPC due to: Service unavailable: Master config (127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [3556ms] refreshing cache from master, [3557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.253:44153, [3558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.254:41699, [3559ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.252:42515, [3560ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.253:44153: Network error: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153, [3561ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.254:41699: Network error: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [3563ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.252:42515: OK, [3564ms] delaying RPC due to: Service unavailable: Master config (127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [5076ms] refreshing cache from master, [5076ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.253:44153, [5077ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.254:41699, [5079ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.252:42515, [5081ms] too many traces: truncated at 100 traces, deferred=Deferred@151931680(state=PENDING, result=null, callback=(continuation of Deferred@611520581 after retry RPC after error@1419081759) -> (continuation of Deferred@952922256 after retry RPC after error@1138781154) -> (continuation of Deferred@2142087907 after retry RPC after error@483465) -> (continuation of Deferred@1528092905 after retry RPC after error@468391698) -> (continuation of Deferred@159603774 after retry RPC after error@851142319) -> (continuation of Deferred@1335178113 after retry RPC after error@210104038) -> (continuation of Deferred@527704548 after retry RPC after error@1989386939) -> (continuation of Deferred@276598483 after retry RPC after error@867016932) -> (continuation of Deferred@233364596 after retry RPC after error@1711612168) -> (continuation of Deferred@318451981 after retry RPC after error@1276053814) -> (continuation of Deferred@122644415 after retry RPC after error@1208818495) -> (continuation of Deferred@53815402 after retry RPC after error@935839619) -> (continuation of Deferred@1263086683 after retry RPC after error@1799160558) -> (continuation of Deferred@1096876443 after retry RPC after error@1217605099) -> (continuation of Deferred@1245379414 after retry RPC after error@1234418317) -> (continuation of Deferred@11995133 after retry RPC after error@1369083509) -> (continuation of Deferred@1281834843 after retry RPC after error@418689928) -> (continuation of Deferred@549771140 after retry RPC after error@623381711) -> (continuation of Deferred@1346396886 after retry RPC after error@1838543781) -> (continuation of Deferred@1802199396 after retry RPC after error@247486130) -> (continuation of Deferred@1573241710 after retry RPC after error@529468328), errback=(continuation of Deferred@611520581 after retry RPC after error@1419081759) -> (continuation of Deferred@952922256 after retry RPC after error@1138781154) -> (continuation of Deferred@2142087907 after retry RPC after error@483465) -> (continuation of Deferred@1528092905 after retry RPC after error@468391698) -> (continuation of Deferred@159603774 after retry RPC after error@851142319) -> (continuation of Deferred@1335178113 after retry RPC after error@210104038) -> (continuation of Deferred@527704548 after retry RPC after error@1989386939) -> (continuation of Deferred@276598483 after retry RPC after error@867016932) -> (continuation of Deferred@233364596 after retry RPC after error@1711612168) -> (continuation of Deferred@318451981 after retry RPC after error@1276053814) -> (continuation of Deferred@122644415 after retry RPC after error@1208818495) -> (continuation of Deferred@53815402 after retry RPC after error@935839619) -> (continuation of Deferred@1263086683 after retry RPC after error@1799160558) -> (continuation of Deferred@1096876443 after retry RPC after error@1217605099) -> (continuation of Deferred@1245379414 after retry RPC after error@1234418317) -> (continuation of Deferred@11995133 after retry RPC after error@1369083509) -> (continuation of Deferred@1281834843 after retry RPC after error@418689928) -> (continuation of Deferred@549771140 after retry RPC after error@623381711) -> (continuation of Deferred@1346396886 after retry RPC after error@1838543781) -> (continuation of Deferred@1802199396 after retry RPC after error@247486130) -> (continuation of Deferred@1573241710 after retry RPC after error@529468328)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.8.112.254:41699
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=22, TimeoutTracker(timeout=30000, elapsed=29829), Traces: [0ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.253:44153, [125ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.254:41699, [136ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.252:42515, [173ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.253:44153: Network error: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153, [172ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.254:41699: Network error: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [590ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.252:42515: OK, [595ms] delaying RPC due to: Service unavailable: Master config (127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [617ms] refreshing cache from master, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.253:44153, [620ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.254:41699, [621ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.252:42515, [630ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.253:44153: Network error: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153, [634ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.254:41699: Network error: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [636ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.252:42515: OK, [637ms] delaying RPC due to: Service unavailable: Master config (127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [657ms] refreshing cache from master, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.253:44153, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.254:41699, [659ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.252:42515, [660ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.253:44153: Network error: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153, [666ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.252:42515: OK, [667ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.254:41699: Network error: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [669ms] delaying RPC due to: Service unavailable: Master config (127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [697ms] refreshing cache from master, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.253:44153, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.254:41699, [700ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.252:42515, [700ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.253:44153: Network error: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153, [706ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.254:41699: Network error: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [708ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.252:42515: OK, [711ms] delaying RPC due to: Service unavailable: Master config (127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [736ms] refreshing cache from master, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.253:44153, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.254:41699, [739ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.253:44153: Network error: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153, [740ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.252:42515, [743ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.254:41699: Network error: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [747ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.252:42515: OK, [748ms] delaying RPC due to: Service unavailable: Master config (127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [776ms] refreshing cache from master, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.253:44153, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.254:41699, [779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.252:42515, [779ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.253:44153: Network error: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153, [789ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.252:42515: OK, [793ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.254:41699: Network error: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [794ms] delaying RPC due to: Service unavailable: Master config (127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [856ms] refreshing cache from master, [857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.253:44153, [859ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.254:41699, [860ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.253:44153: Network error: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153, [860ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.252:42515, [865ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.254:41699: Network error: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [867ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.252:42515: OK, [868ms] delaying RPC due to: Service unavailable: Master config (127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [997ms] refreshing cache from master, [997ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.253:44153, [999ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.254:41699, [999ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.252:42515, [1006ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.252:42515: OK, [1007ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.253:44153: Network error: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153, [1009ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.254:41699: Network error: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [1012ms] delaying RPC due to: Service unavailable: Master config (127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [1196ms] refreshing cache from master, [1197ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.253:44153, [1200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.254:41699, [1200ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.253:44153: Network error: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153, [1202ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.252:42515, [1203ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.254:41699: Network error: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [1208ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.252:42515: OK, [1209ms] delaying RPC due to: Service unavailable: Master config (127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [1416ms] refreshing cache from master, [1417ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.253:44153, [1420ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.254:41699, [1421ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.253:44153: Network error: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153, [1423ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.254:41699: Network error: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [1425ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.252:42515, [1429ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.252:42515: OK, [1431ms] delaying RPC due to: Service unavailable: Master config (127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [2197ms] refreshing cache from master, [2197ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.253:44153, [2198ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.254:41699, [2200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.252:42515, [2200ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.253:44153: Network error: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153, [2204ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.254:41699: Network error: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [2206ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.252:42515: OK, [2206ms] delaying RPC due to: Service unavailable: Master config (127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [3556ms] refreshing cache from master, [3557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.253:44153, [3558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.254:41699, [3559ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.252:42515, [3560ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.253:44153: Network error: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153, [3561ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.254:41699: Network error: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [3563ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.252:42515: OK, [3564ms] delaying RPC due to: Service unavailable: Master config (127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [5076ms] refreshing cache from master, [5076ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.253:44153, [5077ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.254:41699, [5079ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.252:42515, [5081ms] too many traces: truncated at 100 traces, deferred=Deferred@151931680(state=PENDING, result=null, callback=(continuation of Deferred@611520581 after retry RPC after error@1419081759) -> (continuation of Deferred@952922256 after retry RPC after error@1138781154) -> (continuation of Deferred@2142087907 after retry RPC after error@483465) -> (continuation of Deferred@1528092905 after retry RPC after error@468391698) -> (continuation of Deferred@159603774 after retry RPC after error@851142319) -> (continuation of Deferred@1335178113 after retry RPC after error@210104038) -> (continuation of Deferred@527704548 after retry RPC after error@1989386939) -> (continuation of Deferred@276598483 after retry RPC after error@867016932) -> (continuation of Deferred@233364596 after retry RPC after error@1711612168) -> (continuation of Deferred@318451981 after retry RPC after error@1276053814) -> (continuation of Deferred@122644415 after retry RPC after error@1208818495) -> (continuation of Deferred@53815402 after retry RPC after error@935839619) -> (continuation of Deferred@1263086683 after retry RPC after error@1799160558) -> (continuation of Deferred@1096876443 after retry RPC after error@1217605099) -> (continuation of Deferred@1245379414 after retry RPC after error@1234418317) -> (continuation of Deferred@11995133 after retry RPC after error@1369083509) -> (continuation of Deferred@1281834843 after retry RPC after error@418689928) -> (continuation of Deferred@549771140 after retry RPC after error@623381711) -> (continuation of Deferred@1346396886 after retry RPC after error@1838543781) -> (continuation of Deferred@1802199396 after retry RPC after error@247486130) -> (continuation of Deferred@1573241710 after retry RPC after error@529468328), errback=(continuation of Deferred@611520581 after retry RPC after error@1419081759) -> (continuation of Deferred@952922256 after retry RPC after error@1138781154) -> (continuation of Deferred@2142087907 after retry RPC after error@483465) -> (continuation of Deferred@1528092905 after retry RPC after error@468391698) -> (continuation of Deferred@159603774 after retry RPC after error@851142319) -> (continuation of Deferred@1335178113 after retry RPC after error@210104038) -> (continuation of Deferred@527704548 after retry RPC after error@1989386939) -> (continuation of Deferred@276598483 after retry RPC after error@867016932) -> (continuation of Deferred@233364596 after retry RPC after error@1711612168) -> (continuation of Deferred@318451981 after retry RPC after error@1276053814) -> (continuation of Deferred@122644415 after retry RPC after error@1208818495) -> (continuation of Deferred@53815402 after retry RPC after error@935839619) -> (continuation of Deferred@1263086683 after retry RPC after error@1799160558) -> (continuation of Deferred@1096876443 after retry RPC after error@1217605099) -> (continuation of Deferred@1245379414 after retry RPC after error@1234418317) -> (continuation of Deferred@11995133 after retry RPC after error@1369083509) -> (continuation of Deferred@1281834843 after retry RPC after error@418689928) -> (continuation of Deferred@549771140 after retry RPC after error@623381711) -> (continuation of Deferred@1346396886 after retry RPC after error@1838543781) -> (continuation of Deferred@1802199396 after retry RPC after error@247486130) -> (continuation of Deferred@1573241710 after retry RPC after error@529468328)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.8.112.254:41699
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
04:02:03.983 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
04:02:03.983 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=22, TimeoutTracker(timeout=30000, elapsed=29829), Traces: [0ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.253:44153, [125ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.254:41699, [136ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.252:42515, [173ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.253:44153: Network error: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153, [172ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.254:41699: Network error: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [590ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.252:42515: OK, [595ms] delaying RPC due to: Service unavailable: Master config (127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [617ms] refreshing cache from master, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.253:44153, [620ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.254:41699, [621ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.252:42515, [630ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.253:44153: Network error: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153, [634ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.254:41699: Network error: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [636ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.252:42515: OK, [637ms] delaying RPC due to: Service unavailable: Master config (127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [657ms] refreshing cache from master, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.253:44153, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.254:41699, [659ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.252:42515, [660ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.253:44153: Network error: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153, [666ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.252:42515: OK, [667ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.254:41699: Network error: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [669ms] delaying RPC due to: Service unavailable: Master config (127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [697ms] refreshing cache from master, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.253:44153, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.254:41699, [700ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.252:42515, [700ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.253:44153: Network error: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153, [706ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.254:41699: Network error: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [708ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.252:42515: OK, [711ms] delaying RPC due to: Service unavailable: Master config (127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [736ms] refreshing cache from master, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.253:44153, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.254:41699, [739ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.253:44153: Network error: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153, [740ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.252:42515, [743ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.254:41699: Network error: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [747ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.252:42515: OK, [748ms] delaying RPC due to: Service unavailable: Master config (127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [776ms] refreshing cache from master, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.253:44153, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.254:41699, [779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.252:42515, [779ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.253:44153: Network error: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153, [789ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.252:42515: OK, [793ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.254:41699: Network error: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [794ms] delaying RPC due to: Service unavailable: Master config (127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [856ms] refreshing cache from master, [857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.253:44153, [859ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.254:41699, [860ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.253:44153: Network error: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153, [860ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.252:42515, [865ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.254:41699: Network error: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [867ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.252:42515: OK, [868ms] delaying RPC due to: Service unavailable: Master config (127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [997ms] refreshing cache from master, [997ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.253:44153, [999ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.254:41699, [999ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.252:42515, [1006ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.252:42515: OK, [1007ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.253:44153: Network error: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153, [1009ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.254:41699: Network error: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [1012ms] delaying RPC due to: Service unavailable: Master config (127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [1196ms] refreshing cache from master, [1197ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.253:44153, [1200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.254:41699, [1200ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.253:44153: Network error: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153, [1202ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.252:42515, [1203ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.254:41699: Network error: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [1208ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.252:42515: OK, [1209ms] delaying RPC due to: Service unavailable: Master config (127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [1416ms] refreshing cache from master, [1417ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.253:44153, [1420ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.254:41699, [1421ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.253:44153: Network error: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153, [1423ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.254:41699: Network error: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [1425ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.252:42515, [1429ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.252:42515: OK, [1431ms] delaying RPC due to: Service unavailable: Master config (127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [2197ms] refreshing cache from master, [2197ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.253:44153, [2198ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.254:41699, [2200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.252:42515, [2200ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.253:44153: Network error: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153, [2204ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.254:41699: Network error: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [2206ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.252:42515: OK, [2206ms] delaying RPC due to: Service unavailable: Master config (127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [3556ms] refreshing cache from master, [3557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.253:44153, [3558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.254:41699, [3559ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.252:42515, [3560ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.253:44153: Network error: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153, [3561ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.254:41699: Network error: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [3563ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.252:42515: OK, [3564ms] delaying RPC due to: Service unavailable: Master config (127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [5076ms] refreshing cache from master, [5076ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.253:44153, [5077ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.254:41699, [5079ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.252:42515, [5081ms] too many traces: truncated at 100 traces, deferred=Deferred@151931680(state=PENDING, result=null, callback=(continuation of Deferred@611520581 after retry RPC after error@1419081759) -> (continuation of Deferred@952922256 after retry RPC after error@1138781154) -> (continuation of Deferred@2142087907 after retry RPC after error@483465) -> (continuation of Deferred@1528092905 after retry RPC after error@468391698) -> (continuation of Deferred@159603774 after retry RPC after error@851142319) -> (continuation of Deferred@1335178113 after retry RPC after error@210104038) -> (continuation of Deferred@527704548 after retry RPC after error@1989386939) -> (continuation of Deferred@276598483 after retry RPC after error@867016932) -> (continuation of Deferred@233364596 after retry RPC after error@1711612168) -> (continuation of Deferred@318451981 after retry RPC after error@1276053814) -> (continuation of Deferred@122644415 after retry RPC after error@1208818495) -> (continuation of Deferred@53815402 after retry RPC after error@935839619) -> (continuation of Deferred@1263086683 after retry RPC after error@1799160558) -> (continuation of Deferred@1096876443 after retry RPC after error@1217605099) -> (continuation of Deferred@1245379414 after retry RPC after error@1234418317) -> (continuation of Deferred@11995133 after retry RPC after error@1369083509) -> (continuation of Deferred@1281834843 after retry RPC after error@418689928) -> (continuation of Deferred@549771140 after retry RPC after error@623381711) -> (continuation of Deferred@1346396886 after retry RPC after error@1838543781) -> (continuation of Deferred@1802199396 after retry RPC after error@247486130) -> (continuation of Deferred@1573241710 after retry RPC after error@529468328), errback=(continuation of Deferred@611520581 after retry RPC after error@1419081759) -> (continuation of Deferred@952922256 after retry RPC after error@1138781154) -> (continuation of Deferred@2142087907 after retry RPC after error@483465) -> (continuation of Deferred@1528092905 after retry RPC after error@468391698) -> (continuation of Deferred@159603774 after retry RPC after error@851142319) -> (continuation of Deferred@1335178113 after retry RPC after error@210104038) -> (continuation of Deferred@527704548 after retry RPC after error@1989386939) -> (continuation of Deferred@276598483 after retry RPC after error@867016932) -> (continuation of Deferred@233364596 after retry RPC after error@1711612168) -> (continuation of Deferred@318451981 after retry RPC after error@1276053814) -> (continuation of Deferred@122644415 after retry RPC after error@1208818495) -> (continuation of Deferred@53815402 after retry RPC after error@935839619) -> (continuation of Deferred@1263086683 after retry RPC after error@1799160558) -> (continuation of Deferred@1096876443 after retry RPC after error@1217605099) -> (continuation of Deferred@1245379414 after retry RPC after error@1234418317) -> (continuation of Deferred@11995133 after retry RPC after error@1369083509) -> (continuation of Deferred@1281834843 after retry RPC after error@418689928) -> (continuation of Deferred@549771140 after retry RPC after error@623381711) -> (continuation of Deferred@1346396886 after retry RPC after error@1838543781) -> (continuation of Deferred@1802199396 after retry RPC after error@247486130) -> (continuation of Deferred@1573241710 after retry RPC after error@529468328)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.8.112.254:41699
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=22, TimeoutTracker(timeout=30000, elapsed=29829), Traces: [0ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.253:44153, [125ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.254:41699, [136ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.252:42515, [173ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.253:44153: Network error: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153, [172ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.254:41699: Network error: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [590ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.252:42515: OK, [595ms] delaying RPC due to: Service unavailable: Master config (127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [617ms] refreshing cache from master, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.253:44153, [620ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.254:41699, [621ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.252:42515, [630ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.253:44153: Network error: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153, [634ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.254:41699: Network error: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [636ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.252:42515: OK, [637ms] delaying RPC due to: Service unavailable: Master config (127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [657ms] refreshing cache from master, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.253:44153, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.254:41699, [659ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.252:42515, [660ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.253:44153: Network error: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153, [666ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.252:42515: OK, [667ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.254:41699: Network error: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [669ms] delaying RPC due to: Service unavailable: Master config (127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [697ms] refreshing cache from master, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.253:44153, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.254:41699, [700ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.252:42515, [700ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.253:44153: Network error: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153, [706ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.254:41699: Network error: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [708ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.252:42515: OK, [711ms] delaying RPC due to: Service unavailable: Master config (127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [736ms] refreshing cache from master, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.253:44153, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.254:41699, [739ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.253:44153: Network error: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153, [740ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.252:42515, [743ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.254:41699: Network error: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [747ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.252:42515: OK, [748ms] delaying RPC due to: Service unavailable: Master config (127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [776ms] refreshing cache from master, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.253:44153, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.254:41699, [779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.252:42515, [779ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.253:44153: Network error: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153, [789ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.252:42515: OK, [793ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.254:41699: Network error: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [794ms] delaying RPC due to: Service unavailable: Master config (127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [856ms] refreshing cache from master, [857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.253:44153, [859ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.254:41699, [860ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.253:44153: Network error: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153, [860ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.252:42515, [865ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.254:41699: Network error: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [867ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.252:42515: OK, [868ms] delaying RPC due to: Service unavailable: Master config (127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [997ms] refreshing cache from master, [997ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.253:44153, [999ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.254:41699, [999ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.252:42515, [1006ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.252:42515: OK, [1007ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.253:44153: Network error: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153, [1009ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.254:41699: Network error: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [1012ms] delaying RPC due to: Service unavailable: Master config (127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [1196ms] refreshing cache from master, [1197ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.253:44153, [1200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.254:41699, [1200ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.253:44153: Network error: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153, [1202ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.252:42515, [1203ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.254:41699: Network error: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [1208ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.252:42515: OK, [1209ms] delaying RPC due to: Service unavailable: Master config (127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [1416ms] refreshing cache from master, [1417ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.253:44153, [1420ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.254:41699, [1421ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.253:44153: Network error: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153, [1423ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.254:41699: Network error: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [1425ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.252:42515, [1429ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.252:42515: OK, [1431ms] delaying RPC due to: Service unavailable: Master config (127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [2197ms] refreshing cache from master, [2197ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.253:44153, [2198ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.254:41699, [2200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.252:42515, [2200ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.253:44153: Network error: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153, [2204ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.254:41699: Network error: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [2206ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.252:42515: OK, [2206ms] delaying RPC due to: Service unavailable: Master config (127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [3556ms] refreshing cache from master, [3557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.253:44153, [3558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.254:41699, [3559ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.252:42515, [3560ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.253:44153: Network error: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153, [3561ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.254:41699: Network error: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [3563ms] Sub RPC ConnectToMaster: received response from server master-127.8.112.252:42515: OK, [3564ms] delaying RPC due to: Service unavailable: Master config (127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699, [5076ms] refreshing cache from master, [5076ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.253:44153, [5077ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.254:41699, [5079ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.112.252:42515, [5081ms] too many traces: truncated at 100 traces, deferred=Deferred@151931680(state=PENDING, result=null, callback=(continuation of Deferred@611520581 after retry RPC after error@1419081759) -> (continuation of Deferred@952922256 after retry RPC after error@1138781154) -> (continuation of Deferred@2142087907 after retry RPC after error@483465) -> (continuation of Deferred@1528092905 after retry RPC after error@468391698) -> (continuation of Deferred@159603774 after retry RPC after error@851142319) -> (continuation of Deferred@1335178113 after retry RPC after error@210104038) -> (continuation of Deferred@527704548 after retry RPC after error@1989386939) -> (continuation of Deferred@276598483 after retry RPC after error@867016932) -> (continuation of Deferred@233364596 after retry RPC after error@1711612168) -> (continuation of Deferred@318451981 after retry RPC after error@1276053814) -> (continuation of Deferred@122644415 after retry RPC after error@1208818495) -> (continuation of Deferred@53815402 after retry RPC after error@935839619) -> (continuation of Deferred@1263086683 after retry RPC after error@1799160558) -> (continuation of Deferred@1096876443 after retry RPC after error@1217605099) -> (continuation of Deferred@1245379414 after retry RPC after error@1234418317) -> (continuation of Deferred@11995133 after retry RPC after error@1369083509) -> (continuation of Deferred@1281834843 after retry RPC after error@418689928) -> (continuation of Deferred@549771140 after retry RPC after error@623381711) -> (continuation of Deferred@1346396886 after retry RPC after error@1838543781) -> (continuation of Deferred@1802199396 after retry RPC after error@247486130) -> (continuation of Deferred@1573241710 after retry RPC after error@529468328), errback=(continuation of Deferred@611520581 after retry RPC after error@1419081759) -> (continuation of Deferred@952922256 after retry RPC after error@1138781154) -> (continuation of Deferred@2142087907 after retry RPC after error@483465) -> (continuation of Deferred@1528092905 after retry RPC after error@468391698) -> (continuation of Deferred@159603774 after retry RPC after error@851142319) -> (continuation of Deferred@1335178113 after retry RPC after error@210104038) -> (continuation of Deferred@527704548 after retry RPC after error@1989386939) -> (continuation of Deferred@276598483 after retry RPC after error@867016932) -> (continuation of Deferred@233364596 after retry RPC after error@1711612168) -> (continuation of Deferred@318451981 after retry RPC after error@1276053814) -> (continuation of Deferred@122644415 after retry RPC after error@1208818495) -> (continuation of Deferred@53815402 after retry RPC after error@935839619) -> (continuation of Deferred@1263086683 after retry RPC after error@1799160558) -> (continuation of Deferred@1096876443 after retry RPC after error@1217605099) -> (continuation of Deferred@1245379414 after retry RPC after error@1234418317) -> (continuation of Deferred@11995133 after retry RPC after error@1369083509) -> (continuation of Deferred@1281834843 after retry RPC after error@418689928) -> (continuation of Deferred@549771140 after retry RPC after error@623381711) -> (continuation of Deferred@1346396886 after retry RPC after error@1838543781) -> (continuation of Deferred@1802199396 after retry RPC after error@247486130) -> (continuation of Deferred@1573241710 after retry RPC after error@529468328)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.8.112.253:44153,127.8.112.254:41699,127.8.112.252:42515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.253:44153(127.8.112.253:44153): Connection refused: /127.8.112.253:44153,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.112.254:41699(127.8.112.254:41699): Connection refused: /127.8.112.254:41699
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.8.112.254:41699
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)