23:42:03.582 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
23:42:03.594 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskkf9L9h/test-tmp
23:42:03.597 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
23:42:03.600 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskkf9L9h/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
23:42:03.823 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
23:42:04.136 [DEBUG - main] (MiniKuduCluster.java:195) Response:
23:42:04.137 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
23:42:04.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:04.141224 16520 mini_oidc.cc:152] Starting JWKS server
23:42:04.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:04.146698 16520 webserver.cc:460] Webserver started at http://127.0.0.1:41351/ using document root <none> and password file <none>
23:42:04.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:04.147289 16520 mini_oidc.cc:189] Starting OIDC Discovery server
23:42:04.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:04.149820 16520 webserver.cc:460] Webserver started at http://127.0.0.1:36035/ using document root <none> and password file <none>
23:42:04.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-09T23:42:04Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
23:42:04.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-09T23:42:04Z Disabled control of system clock
23:42:04.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:04.214469 16520 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskkf9L9h/build/tsan/bin/kudu
23:42:04.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskkf9L9h/build/tsan/bin/kudu
23:42:04.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/master-0/wal
23:42:04.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/master-0/data
23:42:04.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:42:04.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:42:04.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:42:04.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:42:04.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:42:04.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:42:04.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/master-0/logs
23:42:04.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/master-0/data/info.pb
23:42:04.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:42:04.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:42:04.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:42:04.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:42:04.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:42:04.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:42:04.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:42:04.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:42:04.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
23:42:04.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:42:04.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:42:04.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:42:04.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.34.62:41863
23:42:04.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.34.62
23:42:04.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:42:04.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.34.20:32993
23:42:04.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:42:04.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:42:04.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:42:04.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:42:04.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.34.62:41863,127.16.34.61:42279,127.16.34.60:36581
23:42:04.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:42:04.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41351/jwks.json with env {}
23:42:04.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:04.522917 16534 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:42:04.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:04.523617 16534 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:42:04.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:04.524428 16534 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:42:04.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:04.554352 16534 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
23:42:04.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:04.554708 16534 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:42:04.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:04.555044 16534 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
23:42:04.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:04.555375 16534 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
23:42:04.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:04.555720 16534 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
23:42:04.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:04.556039 16534 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41351/jwks.json
23:42:04.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:04.591500 16534 master_runner.cc:386] Master server non-default flags:
23:42:04.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:42:04.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.34.20:32993
23:42:04.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:42:04.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:42:04.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/master-0/data
23:42:04.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/master-0/wal
23:42:04.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:42:04.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.34.62:41863,127.16.34.61:42279,127.16.34.60:36581
23:42:04.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:42:04.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:42:04.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:42:04.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.34.62:41863
23:42:04.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:42:04.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:42:04.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:42:04.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41351/jwks.json
23:42:04.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:42:04.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:42:04.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/master-0/data/info.pb
23:42:04.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.34.62
23:42:04.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:42:04.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:42:04.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:42:04.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:42:04.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:42:04.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:42:04.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/master-0/logs
23:42:04.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:42:04.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:42:04.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
23:42:04.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
23:42:04.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:42:04.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:42:04.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:42:04.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 20:03:34 UTC on ccycloud.quasar-agiwfo.root.comops.site
23:42:04.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4576543
23:42:04.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:42:04.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:04.593664 16534 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:42:04.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:04.596155 16534 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:42:05.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:04.621012 16540 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:42:05.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:04.622812 16534 server_base.cc:1034] running on GCE node
23:42:05.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:04.621448 16543 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:42:05.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:04.620983 16541 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:42:05.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:05.923574 16534 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:42:05.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:05.927528 16534 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:42:05.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:05.929028 16534 hybrid_clock.cc:648] HybridClock initialized: now 1786318925928983 us; error 65 us; skew 500 ppm
23:42:05.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:05.930126 16534 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:42:05.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:05.936736 16534 webserver.cc:460] Webserver started at http://127.16.34.62:34005/ using document root <none> and password file <none>
23:42:05.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:05.938024 16534 fs_manager.cc:359] Metadata directory not provided
23:42:05.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:05.938216 16534 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:42:05.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:05.938725 16534 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
23:42:05.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:05.944286 16534 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/master-0/data/instance:
23:42:05.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6a9d90cd8a334f448572ee626a8b6c37"
23:42:05.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 23:42:05 on dist-test-slave-8w9v"
23:42:05.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:05.945665 16534 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/master-0/wal/instance:
23:42:05.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6a9d90cd8a334f448572ee626a8b6c37"
23:42:05.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 23:42:05 on dist-test-slave-8w9v"
23:42:05.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:05.954468 16534 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.008s sys 0.000s
23:42:05.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:05.961565 16550 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
23:42:05.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:05.963016 16534 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.002s sys 0.002s
23:42:05.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:05.963531 16534 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/master-0/data,/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/master-0/wal
23:42:05.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6a9d90cd8a334f448572ee626a8b6c37"
23:42:05.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 23:42:05 on dist-test-slave-8w9v"
23:42:05.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:05.963922 16534 fs_report.cc:383] FS layout report
23:42:05.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:42:05.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/master-0/wal
23:42:05.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/master-0/wal
23:42:05.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/master-0/data/data
23:42:05.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:42:05.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:42:05.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:42:05.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:42:05.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:42:05.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:42:05.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:42:05.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:42:05.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:42:05.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:42:05.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:42:06.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:06.005076 16534 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
23:42:06.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:06.006884 16534 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:42:06.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:06.007552 16534 kserver.cc:163] Server-wide thread pool size limit: 3276
23:42:06.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:06.084546 16534 rpc_server.cc:307] RPC server started. Bound to: 127.16.34.62:41863
23:42:06.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:06.084635 16603 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.34.62:41863 every 8 connection(s)
23:42:06.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:06.088020 16534 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/master-0/data/info.pb
23:42:06.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:06.091269 16520 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskkf9L9h/build/tsan/bin/kudu as pid 16534
23:42:06.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:06.092021 16520 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/master-0/wal/instance
23:42:06.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:06.098822 16520 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskkf9L9h/build/tsan/bin/kudu
23:42:06.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskkf9L9h/build/tsan/bin/kudu
23:42:06.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/master-1/wal
23:42:06.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/master-1/data
23:42:06.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:42:06.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:42:06.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:42:06.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:42:06.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:42:06.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:42:06.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/master-1/logs
23:42:06.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/master-1/data/info.pb
23:42:06.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:42:06.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:42:06.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:42:06.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:42:06.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:42:06.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:42:06.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:42:06.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:42:06.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
23:42:06.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:42:06.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:42:06.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:42:06.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.34.61:42279
23:42:06.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.34.61
23:42:06.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:42:06.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.34.20:32993
23:42:06.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:42:06.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:42:06.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:42:06.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:42:06.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.34.62:41863,127.16.34.61:42279,127.16.34.60:36581
23:42:06.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:42:06.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41351/jwks.json with env {}
23:42:06.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:06.099419 16604 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
23:42:06.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:06.117007 16604 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.34.62" port: 41863 } has no permanent_uuid. Determining permanent_uuid...
23:42:06.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:06.151968 16604 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.34.61" port: 42279 } has no permanent_uuid. Determining permanent_uuid...
23:42:06.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:06.154501 16551 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.34.61:42279: connect: Connection refused (error 111)
23:42:06.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:06.158846 16604 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.34.61:42279: Network error: Client connection negotiation failed: client connection to 127.16.34.61:42279: connect: Connection refused (error 111)
23:42:06.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:06.208384 16604 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.34.61" port: 42279 } attempt: 1
23:42:06.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:06.212386 16604 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.34.61:42279: Network error: Client connection negotiation failed: client connection to 127.16.34.61:42279: connect: Connection refused (error 111)
23:42:06.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:06.280766 16604 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.34.61" port: 42279 } attempt: 2
23:42:06.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:06.285053 16604 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.34.61:42279: Network error: Client connection negotiation failed: client connection to 127.16.34.61:42279: connect: Connection refused (error 111)
23:42:06.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:06.376446 16604 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.34.61" port: 42279 } attempt: 3
23:42:06.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:06.380301 16604 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.34.61:42279: Network error: Client connection negotiation failed: client connection to 127.16.34.61:42279: connect: Connection refused (error 111)
23:42:06.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:06.417090 16607 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:42:06.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:06.417771 16607 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:42:06.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:06.418437 16607 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:42:06.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:06.447894 16607 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
23:42:06.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:06.448294 16607 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:42:06.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:06.448616 16607 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
23:42:06.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:06.448977 16607 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
23:42:06.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:06.449303 16607 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
23:42:06.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:06.449606 16607 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41351/jwks.json
23:42:06.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:06.484511 16607 master_runner.cc:386] Master server non-default flags:
23:42:06.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:42:06.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.34.20:32993
23:42:06.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:42:06.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:42:06.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/master-1/data
23:42:06.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/master-1/wal
23:42:06.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:42:06.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.34.62:41863,127.16.34.61:42279,127.16.34.60:36581
23:42:06.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:42:06.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:42:06.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:42:06.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.34.61:42279
23:42:06.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:42:06.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:42:06.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:42:06.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41351/jwks.json
23:42:06.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:42:06.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:42:06.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/master-1/data/info.pb
23:42:06.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.34.61
23:42:06.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:42:06.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:42:06.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:42:06.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:42:06.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:42:06.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:42:06.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/master-1/logs
23:42:06.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:42:06.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:42:06.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
23:42:06.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
23:42:06.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:42:06.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:42:06.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:42:06.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 20:03:34 UTC on ccycloud.quasar-agiwfo.root.comops.site
23:42:06.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4576543
23:42:06.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:42:06.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:06.486089 16607 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:42:06.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:06.488368 16607 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:42:06.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:06.505816 16620 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:42:06.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:06.523660 16604 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.34.61" port: 42279 } attempt: 4
23:42:06.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:06.528039 16604 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.34.61:42279: Network error: Client connection negotiation failed: client connection to 127.16.34.61:42279: connect: Connection refused (error 111)
23:42:06.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:06.827436 16604 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.34.61" port: 42279 } attempt: 5
23:42:06.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:06.831864 16604 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.34.61:42279: Network error: Client connection negotiation failed: client connection to 127.16.34.61:42279: connect: Connection refused (error 111)
23:42:07.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:07.379258 16604 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.34.61" port: 42279 } attempt: 6
23:42:07.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:07.389640 16604 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.34.61:42279: Network error: Client connection negotiation failed: client connection to 127.16.34.61:42279: connect: Connection refused (error 111)
23:42:07.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:06.505954 16619 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:42:07.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:06.506417 16622 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:42:07.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:06.508559 16607 server_base.cc:1034] running on GCE node
23:42:07.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:07.795933 16607 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:42:07.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:07.799396 16607 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:42:07.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:07.800819 16607 hybrid_clock.cc:648] HybridClock initialized: now 1786318927800663 us; error 181 us; skew 500 ppm
23:42:07.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:07.801878 16607 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:42:07.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:07.808440 16607 webserver.cc:460] Webserver started at http://127.16.34.61:41267/ using document root <none> and password file <none>
23:42:07.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:07.809415 16607 fs_manager.cc:359] Metadata directory not provided
23:42:07.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:07.809598 16607 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:42:07.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:07.809981 16607 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
23:42:07.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:07.815255 16607 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/master-1/data/instance:
23:42:07.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1d467dda25e54070a8644675f8b840cd"
23:42:07.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 23:42:07 on dist-test-slave-8w9v"
23:42:07.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:07.816555 16607 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/master-1/wal/instance:
23:42:07.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1d467dda25e54070a8644675f8b840cd"
23:42:07.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 23:42:07 on dist-test-slave-8w9v"
23:42:07.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:07.823802 16607 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.000s
23:42:07.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:07.829236 16631 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
23:42:07.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:07.830330 16607 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
23:42:07.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:07.830598 16607 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/master-1/data,/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/master-1/wal
23:42:07.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1d467dda25e54070a8644675f8b840cd"
23:42:07.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 23:42:07 on dist-test-slave-8w9v"
23:42:07.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:07.830883 16607 fs_report.cc:383] FS layout report
23:42:07.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:42:07.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/master-1/wal
23:42:07.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/master-1/wal
23:42:07.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/master-1/data/data
23:42:07.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:42:07.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:42:07.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:42:07.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:42:07.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:42:07.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:42:07.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:42:07.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:42:07.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:42:07.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:42:07.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:42:07.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:07.865520 16607 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
23:42:07.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:07.866981 16607 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:42:07.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:07.867403 16607 kserver.cc:163] Server-wide thread pool size limit: 3276
23:42:07.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:07.941342 16607 rpc_server.cc:307] RPC server started. Bound to: 127.16.34.61:42279
23:42:07.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:07.941437 16685 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.34.61:42279 every 8 connection(s)
23:42:07.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:07.944181 16607 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/master-1/data/info.pb
23:42:07.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:07.947903 16520 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskkf9L9h/build/tsan/bin/kudu as pid 16607
23:42:07.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:07.948405 16520 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/master-1/wal/instance
23:42:07.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:07.950414 16686 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
23:42:07.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:07.951519 16520 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskkf9L9h/build/tsan/bin/kudu
23:42:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskkf9L9h/build/tsan/bin/kudu
23:42:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/master-2/wal
23:42:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/master-2/data
23:42:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:42:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:42:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:42:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:42:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:42:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:42:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/master-2/logs
23:42:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/master-2/data/info.pb
23:42:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:42:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:42:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:42:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:42:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:42:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:42:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:42:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:42:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
23:42:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:42:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:42:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:42:07.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.34.60:36581
23:42:07.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.34.60
23:42:07.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:42:07.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.34.20:32993
23:42:07.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:42:07.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:42:07.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:42:07.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:42:07.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.34.62:41863,127.16.34.61:42279,127.16.34.60:36581
23:42:07.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:42:07.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41351/jwks.json with env {}
23:42:07.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:07.964623 16686 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.34.62" port: 41863 } has no permanent_uuid. Determining permanent_uuid...
23:42:07.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:07.987306 16686 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.34.61" port: 42279 } has no permanent_uuid. Determining permanent_uuid...
23:42:08.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:08.002519 16686 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.34.60" port: 36581 } has no permanent_uuid. Determining permanent_uuid...
23:42:08.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:08.004649 16634 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.34.60:36581: connect: Connection refused (error 111)
23:42:08.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:08.011386 16686 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.34.60:36581: Network error: Client connection negotiation failed: client connection to 127.16.34.60:36581: connect: Connection refused (error 111)
23:42:08.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:08.060933 16686 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.34.60" port: 36581 } attempt: 1
23:42:08.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:08.065269 16686 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.34.60:36581: Network error: Client connection negotiation failed: client connection to 127.16.34.60:36581: connect: Connection refused (error 111)
23:42:08.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:08.133642 16686 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.34.60" port: 36581 } attempt: 2
23:42:08.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:08.137496 16686 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.34.60:36581: Network error: Client connection negotiation failed: client connection to 127.16.34.60:36581: connect: Connection refused (error 111)
23:42:08.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:08.228911 16686 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.34.60" port: 36581 } attempt: 3
23:42:08.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:08.233196 16686 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.34.60:36581: Network error: Client connection negotiation failed: client connection to 127.16.34.60:36581: connect: Connection refused (error 111)
23:42:08.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:08.265754 16688 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:42:08.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:08.266465 16688 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:42:08.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:08.267156 16688 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:42:08.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:08.298193 16688 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
23:42:08.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:08.298581 16688 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:42:08.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:08.298905 16688 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
23:42:08.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:08.299261 16688 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
23:42:08.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:08.299655 16688 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
23:42:08.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:08.299950 16688 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41351/jwks.json
23:42:08.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:08.336102 16688 master_runner.cc:386] Master server non-default flags:
23:42:08.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:42:08.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.34.20:32993
23:42:08.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:42:08.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:42:08.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/master-2/data
23:42:08.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/master-2/wal
23:42:08.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:42:08.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.34.62:41863,127.16.34.61:42279,127.16.34.60:36581
23:42:08.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:42:08.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:42:08.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:42:08.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.34.60:36581
23:42:08.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:42:08.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:42:08.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:42:08.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41351/jwks.json
23:42:08.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:42:08.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:42:08.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/master-2/data/info.pb
23:42:08.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.34.60
23:42:08.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:42:08.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:42:08.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:42:08.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:42:08.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:42:08.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:42:08.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/master-2/logs
23:42:08.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:42:08.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:42:08.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
23:42:08.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
23:42:08.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:42:08.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:42:08.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:42:08.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 20:03:34 UTC on ccycloud.quasar-agiwfo.root.comops.site
23:42:08.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4576543
23:42:08.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:42:08.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:08.337826 16688 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:42:08.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:08.340170 16688 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:42:08.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:08.359556 16701 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:42:08.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:08.376560 16686 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.34.60" port: 36581 } attempt: 4
23:42:08.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:08.380203 16686 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.34.60:36581: Network error: Client connection negotiation failed: client connection to 127.16.34.60:36581: connect: Connection refused (error 111)
23:42:08.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:08.450688 16604 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.34.61" port: 42279 } attempt: 7
23:42:08.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:08.466710 16604 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.34.60" port: 36581 } has no permanent_uuid. Determining permanent_uuid...
23:42:08.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:08.473321 16604 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.34.60:36581: Network error: Client connection negotiation failed: client connection to 127.16.34.60:36581: connect: Connection refused (error 111)
23:42:08.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:08.531742 16604 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.34.60" port: 36581 } attempt: 1
23:42:08.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:08.535434 16604 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.34.60:36581: Network error: Client connection negotiation failed: client connection to 127.16.34.60:36581: connect: Connection refused (error 111)
23:42:08.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:08.617056 16604 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.34.60" port: 36581 } attempt: 2
23:42:08.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:08.621206 16604 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.34.60:36581: Network error: Client connection negotiation failed: client connection to 127.16.34.60:36581: connect: Connection refused (error 111)
23:42:08.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:08.679863 16686 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.34.60" port: 36581 } attempt: 5
23:42:08.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:08.684466 16686 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.34.60:36581: Network error: Client connection negotiation failed: client connection to 127.16.34.60:36581: connect: Connection refused (error 111)
23:42:08.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:08.706779 16604 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.34.60" port: 36581 } attempt: 3
23:42:08.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:08.711158 16604 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.34.60:36581: Network error: Client connection negotiation failed: client connection to 127.16.34.60:36581: connect: Connection refused (error 111)
23:42:08.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:08.851536 16604 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.34.60" port: 36581 } attempt: 4
23:42:08.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:08.855837 16604 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.34.60:36581: Network error: Client connection negotiation failed: client connection to 127.16.34.60:36581: connect: Connection refused (error 111)
23:42:09.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.139273 16604 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.34.60" port: 36581 } attempt: 5
23:42:09.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:09.143921 16604 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.34.60:36581: Network error: Client connection negotiation failed: client connection to 127.16.34.60:36581: connect: Connection refused (error 111)
23:42:09.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.231921 16686 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.34.60" port: 36581 } attempt: 6
23:42:09.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:09.239727 16686 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.34.60:36581: Network error: Client connection negotiation failed: client connection to 127.16.34.60:36581: connect: Connection refused (error 111)
23:42:09.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:08.359645 16698 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:42:09.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:08.359709 16699 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:42:09.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:08.361359 16688 server_base.cc:1034] running on GCE node
23:42:09.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.551065 16688 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:42:09.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.553767 16688 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:42:09.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.555094 16688 hybrid_clock.cc:648] HybridClock initialized: now 1786318929555062 us; error 50 us; skew 500 ppm
23:42:09.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.555869 16688 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:42:09.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.563591 16688 webserver.cc:460] Webserver started at http://127.16.34.60:46487/ using document root <none> and password file <none>
23:42:09.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.564568 16688 fs_manager.cc:359] Metadata directory not provided
23:42:09.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.564747 16688 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:42:09.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.565138 16688 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
23:42:09.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.570020 16688 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/master-2/data/instance:
23:42:09.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6fe54b56d7fb451e817d34984c697a86"
23:42:09.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 23:42:09 on dist-test-slave-8w9v"
23:42:09.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.571241 16688 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/master-2/wal/instance:
23:42:09.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6fe54b56d7fb451e817d34984c697a86"
23:42:09.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 23:42:09 on dist-test-slave-8w9v"
23:42:09.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.580595 16688 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.006s sys 0.005s
23:42:09.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.587918 16713 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
23:42:09.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.589380 16688 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.002s sys 0.001s
23:42:09.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.589771 16688 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/master-2/data,/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/master-2/wal
23:42:09.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6fe54b56d7fb451e817d34984c697a86"
23:42:09.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 23:42:09 on dist-test-slave-8w9v"
23:42:09.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.590191 16688 fs_report.cc:383] FS layout report
23:42:09.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:42:09.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/master-2/wal
23:42:09.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/master-2/wal
23:42:09.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/master-2/data/data
23:42:09.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:42:09.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:42:09.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:42:09.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:42:09.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:42:09.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:42:09.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:42:09.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:42:09.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:42:09.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:42:09.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:42:09.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.623862 16688 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
23:42:09.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.625262 16688 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:42:09.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.625648 16688 kserver.cc:163] Server-wide thread pool size limit: 3276
23:42:09.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.695662 16688 rpc_server.cc:307] RPC server started. Bound to: 127.16.34.60:36581
23:42:09.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.695761 16767 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.34.60:36581 every 8 connection(s)
23:42:09.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.696357 16604 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.34.60" port: 36581 } attempt: 6
23:42:09.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.698534 16688 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/master-2/data/info.pb
23:42:09.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.702148 16520 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskkf9L9h/build/tsan/bin/kudu as pid 16688
23:42:09.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.702582 16520 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/master-2/wal/instance
23:42:09.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.705343 16770 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
23:42:09.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.737615 16770 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.34.62" port: 41863 } has no permanent_uuid. Determining permanent_uuid...
23:42:09.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.750108 16604 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6a9d90cd8a334f448572ee626a8b6c37: Bootstrap starting.
23:42:09.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.762382 16604 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 6a9d90cd8a334f448572ee626a8b6c37: Neither blocks nor log segments found. Creating new log.
23:42:09.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.766142 16604 log.cc:824] T 00000000000000000000000000000000 P 6a9d90cd8a334f448572ee626a8b6c37: Log is configured to *not* fsync() on all Append() calls
23:42:09.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.771930 16604 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6a9d90cd8a334f448572ee626a8b6c37: No bootstrap required, opened a new log
23:42:09.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.776041 16770 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.34.61" port: 42279 } has no permanent_uuid. Determining permanent_uuid...
23:42:09.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.788576 16770 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.34.60" port: 36581 } has no permanent_uuid. Determining permanent_uuid...
23:42:09.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.799664 16604 raft_consensus.cc:348] T 00000000000000000000000000000000 P 6a9d90cd8a334f448572ee626a8b6c37 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a9d90cd8a334f448572ee626a8b6c37" member_type: VOTER last_known_addr { host: "127.16.34.62" port: 41863 } } peers { permanent_uuid: "1d467dda25e54070a8644675f8b840cd" member_type: VOTER last_known_addr { host: "127.16.34.61" port: 42279 } } peers { permanent_uuid: "6fe54b56d7fb451e817d34984c697a86" member_type: VOTER last_known_addr { host: "127.16.34.60" port: 36581 } }
23:42:09.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.801367 16604 raft_consensus.cc:374] T 00000000000000000000000000000000 P 6a9d90cd8a334f448572ee626a8b6c37 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
23:42:09.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.801708 16604 raft_consensus.cc:729] T 00000000000000000000000000000000 P 6a9d90cd8a334f448572ee626a8b6c37 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a9d90cd8a334f448572ee626a8b6c37, State: Initialized, Role: FOLLOWER
23:42:09.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.803030 16604 consensus_queue.cc:260] T 00000000000000000000000000000000 P 6a9d90cd8a334f448572ee626a8b6c37 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a9d90cd8a334f448572ee626a8b6c37" member_type: VOTER last_known_addr { host: "127.16.34.62" port: 41863 } } peers { permanent_uuid: "1d467dda25e54070a8644675f8b840cd" member_type: VOTER last_known_addr { host: "127.16.34.61" port: 42279 } } peers { permanent_uuid: "6fe54b56d7fb451e817d34984c697a86" member_type: VOTER last_known_addr { host: "127.16.34.60" port: 36581 } }
23:42:09.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.807906 16770 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6fe54b56d7fb451e817d34984c697a86: Bootstrap starting.
23:42:09.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.814848 16779 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6a9d90cd8a334f448572ee626a8b6c37 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a9d90cd8a334f448572ee626a8b6c37" member_type: VOTER last_known_addr { host: "127.16.34.62" port: 41863 } } peers { permanent_uuid: "1d467dda25e54070a8644675f8b840cd" member_type: VOTER last_known_addr { host: "127.16.34.61" port: 42279 } } peers { permanent_uuid: "6fe54b56d7fb451e817d34984c697a86" member_type: VOTER last_known_addr { host: "127.16.34.60" port: 36581 } } }
23:42:09.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.816071 16770 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 6fe54b56d7fb451e817d34984c697a86: Neither blocks nor log segments found. Creating new log.
23:42:09.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.816071 16779 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6a9d90cd8a334f448572ee626a8b6c37 [sys.catalog]: This master's current role is: FOLLOWER
23:42:09.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.817044 16604 sys_catalog.cc:564] T 00000000000000000000000000000000 P 6a9d90cd8a334f448572ee626a8b6c37 [sys.catalog]: configured and running, proceeding with master startup.
23:42:09.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.818820 16770 log.cc:824] T 00000000000000000000000000000000 P 6fe54b56d7fb451e817d34984c697a86: Log is configured to *not* fsync() on all Append() calls
23:42:09.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.829952 16770 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6fe54b56d7fb451e817d34984c697a86: No bootstrap required, opened a new log
23:42:09.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.847218 16779 raft_consensus.cc:482] T 00000000000000000000000000000000 P 6a9d90cd8a334f448572ee626a8b6c37 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
23:42:09.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.847709 16779 raft_consensus.cc:504] T 00000000000000000000000000000000 P 6a9d90cd8a334f448572ee626a8b6c37 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a9d90cd8a334f448572ee626a8b6c37" member_type: VOTER last_known_addr { host: "127.16.34.62" port: 41863 } } peers { permanent_uuid: "1d467dda25e54070a8644675f8b840cd" member_type: VOTER last_known_addr { host: "127.16.34.61" port: 42279 } } peers { permanent_uuid: "6fe54b56d7fb451e817d34984c697a86" member_type: VOTER last_known_addr { host: "127.16.34.60" port: 36581 } }
23:42:09.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.850721 16779 leader_election.cc:288] T 00000000000000000000000000000000 P 6a9d90cd8a334f448572ee626a8b6c37 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1d467dda25e54070a8644675f8b840cd (127.16.34.61:42279), 6fe54b56d7fb451e817d34984c697a86 (127.16.34.60:36581)
23:42:09.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:09.864573 16794 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 6a9d90cd8a334f448572ee626a8b6c37: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:42:09.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:09.865319 16794 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:42:09.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.873579 16770 raft_consensus.cc:348] T 00000000000000000000000000000000 P 6fe54b56d7fb451e817d34984c697a86 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a9d90cd8a334f448572ee626a8b6c37" member_type: VOTER last_known_addr { host: "127.16.34.62" port: 41863 } } peers { permanent_uuid: "1d467dda25e54070a8644675f8b840cd" member_type: VOTER last_known_addr { host: "127.16.34.61" port: 42279 } } peers { permanent_uuid: "6fe54b56d7fb451e817d34984c697a86" member_type: VOTER last_known_addr { host: "127.16.34.60" port: 36581 } }
23:42:09.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.876185 16770 raft_consensus.cc:374] T 00000000000000000000000000000000 P 6fe54b56d7fb451e817d34984c697a86 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
23:42:09.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.877139 16770 raft_consensus.cc:729] T 00000000000000000000000000000000 P 6fe54b56d7fb451e817d34984c697a86 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6fe54b56d7fb451e817d34984c697a86, State: Initialized, Role: FOLLOWER
23:42:09.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.879608 16770 consensus_queue.cc:260] T 00000000000000000000000000000000 P 6fe54b56d7fb451e817d34984c697a86 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a9d90cd8a334f448572ee626a8b6c37" member_type: VOTER last_known_addr { host: "127.16.34.62" port: 41863 } } peers { permanent_uuid: "1d467dda25e54070a8644675f8b840cd" member_type: VOTER last_known_addr { host: "127.16.34.61" port: 42279 } } peers { permanent_uuid: "6fe54b56d7fb451e817d34984c697a86" member_type: VOTER last_known_addr { host: "127.16.34.60" port: 36581 } }
23:42:09.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.855217 16534 master_runner.cc:186] Error getting master registration for 127.16.34.61:42279: OK, instance_id { permanent_uuid: "1d467dda25e54070a8644675f8b840cd" instance_seqno: 1786318927921385 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
23:42:09.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.882673 16534 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
23:42:09.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.885212 16796 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6fe54b56d7fb451e817d34984c697a86 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a9d90cd8a334f448572ee626a8b6c37" member_type: VOTER last_known_addr { host: "127.16.34.62" port: 41863 } } peers { permanent_uuid: "1d467dda25e54070a8644675f8b840cd" member_type: VOTER last_known_addr { host: "127.16.34.61" port: 42279 } } peers { permanent_uuid: "6fe54b56d7fb451e817d34984c697a86" member_type: VOTER last_known_addr { host: "127.16.34.60" port: 36581 } } }
23:42:09.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.886168 16796 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6fe54b56d7fb451e817d34984c697a86 [sys.catalog]: This master's current role is: FOLLOWER
23:42:09.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.886878 16770 sys_catalog.cc:564] T 00000000000000000000000000000000 P 6fe54b56d7fb451e817d34984c697a86 [sys.catalog]: configured and running, proceeding with master startup.
23:42:09.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.887079 16741 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6a9d90cd8a334f448572ee626a8b6c37" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6fe54b56d7fb451e817d34984c697a86" is_pre_election: true
23:42:09.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.888195 16741 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 6fe54b56d7fb451e817d34984c697a86 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6a9d90cd8a334f448572ee626a8b6c37 in term 0.
23:42:09.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.889619 16553 leader_election.cc:302] T 00000000000000000000000000000000 P 6a9d90cd8a334f448572ee626a8b6c37 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a9d90cd8a334f448572ee626a8b6c37, 6fe54b56d7fb451e817d34984c697a86; no voters:
23:42:09.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.890661 16779 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 6a9d90cd8a334f448572ee626a8b6c37 [term 0 FOLLOWER]: Leader pre-election won for term 1
23:42:09.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.890976 16779 raft_consensus.cc:482] T 00000000000000000000000000000000 P 6a9d90cd8a334f448572ee626a8b6c37 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
23:42:09.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.891273 16779 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 6a9d90cd8a334f448572ee626a8b6c37 [term 0 FOLLOWER]: Advancing to term 1
23:42:09.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.896407 16779 raft_consensus.cc:504] T 00000000000000000000000000000000 P 6a9d90cd8a334f448572ee626a8b6c37 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a9d90cd8a334f448572ee626a8b6c37" member_type: VOTER last_known_addr { host: "127.16.34.62" port: 41863 } } peers { permanent_uuid: "1d467dda25e54070a8644675f8b840cd" member_type: VOTER last_known_addr { host: "127.16.34.61" port: 42279 } } peers { permanent_uuid: "6fe54b56d7fb451e817d34984c697a86" member_type: VOTER last_known_addr { host: "127.16.34.60" port: 36581 } }
23:42:09.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.896639 16796 raft_consensus.cc:482] T 00000000000000000000000000000000 P 6fe54b56d7fb451e817d34984c697a86 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
23:42:09.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.897393 16796 raft_consensus.cc:504] T 00000000000000000000000000000000 P 6fe54b56d7fb451e817d34984c697a86 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a9d90cd8a334f448572ee626a8b6c37" member_type: VOTER last_known_addr { host: "127.16.34.62" port: 41863 } } peers { permanent_uuid: "1d467dda25e54070a8644675f8b840cd" member_type: VOTER last_known_addr { host: "127.16.34.61" port: 42279 } } peers { permanent_uuid: "6fe54b56d7fb451e817d34984c697a86" member_type: VOTER last_known_addr { host: "127.16.34.60" port: 36581 } }
23:42:09.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.902006 16796 leader_election.cc:288] T 00000000000000000000000000000000 P 6fe54b56d7fb451e817d34984c697a86 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6a9d90cd8a334f448572ee626a8b6c37 (127.16.34.62:41863), 1d467dda25e54070a8644675f8b840cd (127.16.34.61:42279)
23:42:09.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.903093 16741 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6a9d90cd8a334f448572ee626a8b6c37" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6fe54b56d7fb451e817d34984c697a86"
23:42:09.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.903092 16578 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6fe54b56d7fb451e817d34984c697a86" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a9d90cd8a334f448572ee626a8b6c37" is_pre_election: true
23:42:09.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.903685 16741 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 6fe54b56d7fb451e817d34984c697a86 [term 0 FOLLOWER]: Advancing to term 1
23:42:09.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.904404 16578 raft_consensus.cc:2379] T 00000000000000000000000000000000 P 6a9d90cd8a334f448572ee626a8b6c37 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 6fe54b56d7fb451e817d34984c697a86 in current term 1: Already voted for candidate 6a9d90cd8a334f448572ee626a8b6c37 in this term.
23:42:09.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.903167 16779 leader_election.cc:288] T 00000000000000000000000000000000 P 6a9d90cd8a334f448572ee626a8b6c37 [CANDIDATE]: Term 1 election: Requested vote from peers 1d467dda25e54070a8644675f8b840cd (127.16.34.61:42279), 6fe54b56d7fb451e817d34984c697a86 (127.16.34.60:36581)
23:42:09.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.912330 16741 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 6fe54b56d7fb451e817d34984c697a86 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6a9d90cd8a334f448572ee626a8b6c37 in term 1.
23:42:09.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.914543 16553 leader_election.cc:302] T 00000000000000000000000000000000 P 6a9d90cd8a334f448572ee626a8b6c37 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6a9d90cd8a334f448572ee626a8b6c37, 6fe54b56d7fb451e817d34984c697a86; no voters:
23:42:09.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.915401 16779 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 6a9d90cd8a334f448572ee626a8b6c37 [term 1 FOLLOWER]: Leader election won for term 1
23:42:09.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.925931 16779 raft_consensus.cc:686] T 00000000000000000000000000000000 P 6a9d90cd8a334f448572ee626a8b6c37 [term 1 LEADER]: Becoming Leader. State: Replica: 6a9d90cd8a334f448572ee626a8b6c37, State: Running, Role: LEADER
23:42:09.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.926985 16779 consensus_queue.cc:237] T 00000000000000000000000000000000 P 6a9d90cd8a334f448572ee626a8b6c37 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a9d90cd8a334f448572ee626a8b6c37" member_type: VOTER last_known_addr { host: "127.16.34.62" port: 41863 } } peers { permanent_uuid: "1d467dda25e54070a8644675f8b840cd" member_type: VOTER last_known_addr { host: "127.16.34.61" port: 42279 } } peers { permanent_uuid: "6fe54b56d7fb451e817d34984c697a86" member_type: VOTER last_known_addr { host: "127.16.34.60" port: 36581 } }
23:42:09.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.947315 16801 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6a9d90cd8a334f448572ee626a8b6c37 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 6a9d90cd8a334f448572ee626a8b6c37. Latest consensus state: current_term: 1 leader_uuid: "6a9d90cd8a334f448572ee626a8b6c37" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a9d90cd8a334f448572ee626a8b6c37" member_type: VOTER last_known_addr { host: "127.16.34.62" port: 41863 } } peers { permanent_uuid: "1d467dda25e54070a8644675f8b840cd" member_type: VOTER last_known_addr { host: "127.16.34.61" port: 42279 } } peers { permanent_uuid: "6fe54b56d7fb451e817d34984c697a86" member_type: VOTER last_known_addr { host: "127.16.34.60" port: 36581 } } }
23:42:09.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.948154 16801 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6a9d90cd8a334f448572ee626a8b6c37 [sys.catalog]: This master's current role is: LEADER
23:42:09.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.974336 16810 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
23:42:09.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.981622 16807 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
23:42:09.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.989729 16807 catalog_manager.cc:1484] Initializing Kudu cluster ID...
23:42:10.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:10.019124 16556 tablet.cc:2369] T 00000000000000000000000000000000 P 6a9d90cd8a334f448572ee626a8b6c37: Can't schedule compaction. Clean time has not been advanced past its initial value.
23:42:10.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:10.019948 16741 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 6fe54b56d7fb451e817d34984c697a86 [term 1 FOLLOWER]: Refusing update from remote peer 6a9d90cd8a334f448572ee626a8b6c37: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
23:42:10.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:10.021770 16801 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 6a9d90cd8a334f448572ee626a8b6c37 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6fe54b56d7fb451e817d34984c697a86" member_type: VOTER last_known_addr { host: "127.16.34.60" port: 36581 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
23:42:10.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:09.987941 16688 master_runner.cc:186] Error getting master registration for 127.16.34.61:42279: OK, instance_id { permanent_uuid: "1d467dda25e54070a8644675f8b840cd" instance_seqno: 1786318927921385 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
23:42:10.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:10.028515 16688 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
23:42:10.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:10.068483 16811 mvcc.cc:205] Tried to move back new op lower bound from 7316762337342627840 to 7316762336991944704. Current Snapshot: MvccSnapshot[applied={T|T < 7316762337342627840}]
23:42:10.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:10.071177 16796 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6fe54b56d7fb451e817d34984c697a86 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 6a9d90cd8a334f448572ee626a8b6c37. Latest consensus state: current_term: 1 leader_uuid: "6a9d90cd8a334f448572ee626a8b6c37" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a9d90cd8a334f448572ee626a8b6c37" member_type: VOTER last_known_addr { host: "127.16.34.62" port: 41863 } } peers { permanent_uuid: "1d467dda25e54070a8644675f8b840cd" member_type: VOTER last_known_addr { host: "127.16.34.61" port: 42279 } } peers { permanent_uuid: "6fe54b56d7fb451e817d34984c697a86" member_type: VOTER last_known_addr { host: "127.16.34.60" port: 36581 } } }
23:42:10.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:10.073617 16796 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6fe54b56d7fb451e817d34984c697a86 [sys.catalog]: This master's current role is: FOLLOWER
23:42:10.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:10.081214 16796 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6fe54b56d7fb451e817d34984c697a86 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "6a9d90cd8a334f448572ee626a8b6c37" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a9d90cd8a334f448572ee626a8b6c37" member_type: VOTER last_known_addr { host: "127.16.34.62" port: 41863 } } peers { permanent_uuid: "1d467dda25e54070a8644675f8b840cd" member_type: VOTER last_known_addr { host: "127.16.34.61" port: 42279 } } peers { permanent_uuid: "6fe54b56d7fb451e817d34984c697a86" member_type: VOTER last_known_addr { host: "127.16.34.60" port: 36581 } } }
23:42:10.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:10.082242 16796 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6fe54b56d7fb451e817d34984c697a86 [sys.catalog]: This master's current role is: FOLLOWER
23:42:10.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:10.085489 16779 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6a9d90cd8a334f448572ee626a8b6c37 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "6a9d90cd8a334f448572ee626a8b6c37" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a9d90cd8a334f448572ee626a8b6c37" member_type: VOTER last_known_addr { host: "127.16.34.62" port: 41863 } } peers { permanent_uuid: "1d467dda25e54070a8644675f8b840cd" member_type: VOTER last_known_addr { host: "127.16.34.61" port: 42279 } } peers { permanent_uuid: "6fe54b56d7fb451e817d34984c697a86" member_type: VOTER last_known_addr { host: "127.16.34.60" port: 36581 } } }
23:42:10.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:10.086184 16779 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6a9d90cd8a334f448572ee626a8b6c37 [sys.catalog]: This master's current role is: LEADER
23:42:10.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:10.090153 16807 catalog_manager.cc:1347] Generated new cluster ID: 31c0efa5fe4a4c4da78972b2fb482efd
23:42:10.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:10.090453 16807 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
23:42:10.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:10.151394 16807 catalog_manager.cc:1370] Generated new certificate authority record
23:42:10.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:10.155174 16807 catalog_manager.cc:1504] Loading token signing keys...
23:42:10.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:10.185890 16807 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 6a9d90cd8a334f448572ee626a8b6c37: Generated new TSK 0
23:42:10.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:10.187502 16807 catalog_manager.cc:1514] Initializing in-progress tserver states...
23:42:10.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:10.221865 16520 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskkf9L9h/build/tsan/bin/kudu
23:42:10.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskkf9L9h/build/tsan/bin/kudu
23:42:10.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/ts-0/wal
23:42:10.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/ts-0/data
23:42:10.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:42:10.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:42:10.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:42:10.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:42:10.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:42:10.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:42:10.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/ts-0/logs
23:42:10.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/ts-0/data/info.pb
23:42:10.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:42:10.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:42:10.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:42:10.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:42:10.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:42:10.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:42:10.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:42:10.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:42:10.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:42:10.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:42:10.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.34.1:0
23:42:10.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.34.1
23:42:10.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.34.1
23:42:10.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:42:10.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.34.62:41863,127.16.34.61:42279,127.16.34.60:36581
23:42:10.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.34.20:32993
23:42:10.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:42:10.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:42:10.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:42:10.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:10.300163 16686 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.34.60" port: 36581 } attempt: 7
23:42:10.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:10.336956 16686 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1d467dda25e54070a8644675f8b840cd: Bootstrap starting.
23:42:10.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:10.344044 16686 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 1d467dda25e54070a8644675f8b840cd: Neither blocks nor log segments found. Creating new log.
23:42:10.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:10.346400 16686 log.cc:824] T 00000000000000000000000000000000 P 1d467dda25e54070a8644675f8b840cd: Log is configured to *not* fsync() on all Append() calls
23:42:10.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:10.356892 16686 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1d467dda25e54070a8644675f8b840cd: No bootstrap required, opened a new log
23:42:10.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:10.388963 16686 raft_consensus.cc:348] T 00000000000000000000000000000000 P 1d467dda25e54070a8644675f8b840cd [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a9d90cd8a334f448572ee626a8b6c37" member_type: VOTER last_known_addr { host: "127.16.34.62" port: 41863 } } peers { permanent_uuid: "1d467dda25e54070a8644675f8b840cd" member_type: VOTER last_known_addr { host: "127.16.34.61" port: 42279 } } peers { permanent_uuid: "6fe54b56d7fb451e817d34984c697a86" member_type: VOTER last_known_addr { host: "127.16.34.60" port: 36581 } }
23:42:10.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:10.389871 16686 raft_consensus.cc:374] T 00000000000000000000000000000000 P 1d467dda25e54070a8644675f8b840cd [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
23:42:10.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:10.390132 16686 raft_consensus.cc:729] T 00000000000000000000000000000000 P 1d467dda25e54070a8644675f8b840cd [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1d467dda25e54070a8644675f8b840cd, State: Initialized, Role: FOLLOWER
23:42:10.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:10.391144 16686 consensus_queue.cc:260] T 00000000000000000000000000000000 P 1d467dda25e54070a8644675f8b840cd [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a9d90cd8a334f448572ee626a8b6c37" member_type: VOTER last_known_addr { host: "127.16.34.62" port: 41863 } } peers { permanent_uuid: "1d467dda25e54070a8644675f8b840cd" member_type: VOTER last_known_addr { host: "127.16.34.61" port: 42279 } } peers { permanent_uuid: "6fe54b56d7fb451e817d34984c697a86" member_type: VOTER last_known_addr { host: "127.16.34.60" port: 36581 } }
23:42:10.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:10.399297 16822 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1d467dda25e54070a8644675f8b840cd [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a9d90cd8a334f448572ee626a8b6c37" member_type: VOTER last_known_addr { host: "127.16.34.62" port: 41863 } } peers { permanent_uuid: "1d467dda25e54070a8644675f8b840cd" member_type: VOTER last_known_addr { host: "127.16.34.61" port: 42279 } } peers { permanent_uuid: "6fe54b56d7fb451e817d34984c697a86" member_type: VOTER last_known_addr { host: "127.16.34.60" port: 36581 } } }
23:42:10.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:10.400143 16822 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1d467dda25e54070a8644675f8b840cd [sys.catalog]: This master's current role is: FOLLOWER
23:42:10.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:10.402942 16686 sys_catalog.cc:564] T 00000000000000000000000000000000 P 1d467dda25e54070a8644675f8b840cd [sys.catalog]: configured and running, proceeding with master startup.
23:42:10.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:10.404054 16656 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 1d467dda25e54070a8644675f8b840cd [term 0 FOLLOWER]: Advancing to term 1
23:42:10.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:10.404748 16659 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6a9d90cd8a334f448572ee626a8b6c37" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1d467dda25e54070a8644675f8b840cd" is_pre_election: true
23:42:10.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:10.405501 16658 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6a9d90cd8a334f448572ee626a8b6c37" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1d467dda25e54070a8644675f8b840cd"
23:42:10.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:10.403118 16657 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6fe54b56d7fb451e817d34984c697a86" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1d467dda25e54070a8644675f8b840cd" is_pre_election: true
23:42:10.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:10.414016 16714 leader_election.cc:302] T 00000000000000000000000000000000 P 6fe54b56d7fb451e817d34984c697a86 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6fe54b56d7fb451e817d34984c697a86; no voters: 1d467dda25e54070a8644675f8b840cd, 6a9d90cd8a334f448572ee626a8b6c37
23:42:10.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:10.415091 16796 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 6fe54b56d7fb451e817d34984c697a86 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
23:42:10.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:10.421953 16656 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 1d467dda25e54070a8644675f8b840cd [term 1 FOLLOWER]: Refusing update from remote peer 6a9d90cd8a334f448572ee626a8b6c37: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
23:42:10.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:10.423918 16817 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 6a9d90cd8a334f448572ee626a8b6c37 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1d467dda25e54070a8644675f8b840cd" member_type: VOTER last_known_addr { host: "127.16.34.61" port: 42279 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
23:42:10.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:10.489944 16830 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1d467dda25e54070a8644675f8b840cd [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "6a9d90cd8a334f448572ee626a8b6c37" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a9d90cd8a334f448572ee626a8b6c37" member_type: VOTER last_known_addr { host: "127.16.34.62" port: 41863 } } peers { permanent_uuid: "1d467dda25e54070a8644675f8b840cd" member_type: VOTER last_known_addr { host: "127.16.34.61" port: 42279 } } peers { permanent_uuid: "6fe54b56d7fb451e817d34984c697a86" member_type: VOTER last_known_addr { host: "127.16.34.60" port: 36581 } } }
23:42:10.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:10.492421 16830 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1d467dda25e54070a8644675f8b840cd [sys.catalog]: This master's current role is: FOLLOWER
23:42:10.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:10.491520 16822 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1d467dda25e54070a8644675f8b840cd [sys.catalog]: SysCatalogTable state changed. Reason: New leader 6a9d90cd8a334f448572ee626a8b6c37. Latest consensus state: current_term: 1 leader_uuid: "6a9d90cd8a334f448572ee626a8b6c37" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a9d90cd8a334f448572ee626a8b6c37" member_type: VOTER last_known_addr { host: "127.16.34.62" port: 41863 } } peers { permanent_uuid: "1d467dda25e54070a8644675f8b840cd" member_type: VOTER last_known_addr { host: "127.16.34.61" port: 42279 } } peers { permanent_uuid: "6fe54b56d7fb451e817d34984c697a86" member_type: VOTER last_known_addr { host: "127.16.34.60" port: 36581 } } }
23:42:10.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:10.493386 16822 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1d467dda25e54070a8644675f8b840cd [sys.catalog]: This master's current role is: FOLLOWER
23:42:10.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:10.524736 16838 catalog_manager.cc:1259] Loaded cluster ID: 31c0efa5fe4a4c4da78972b2fb482efd
23:42:10.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:10.524953 16838 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 1d467dda25e54070a8644675f8b840cd: loading cluster ID for follower catalog manager: success
23:42:10.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:10.530362 16838 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 1d467dda25e54070a8644675f8b840cd: acquiring CA information for follower catalog manager: success
23:42:10.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:10.533906 16838 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 1d467dda25e54070a8644675f8b840cd: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
23:42:10.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:10.711768 16818 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:42:10.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:10.712474 16818 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:42:10.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:10.713234 16818 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:42:10.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:10.768635 16818 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:42:10.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:10.770018 16818 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.34.1
23:42:10.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:10.829833 16818 tablet_server_runner.cc:78] Tablet server non-default flags:
23:42:10.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:42:10.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.34.20:32993
23:42:10.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:42:10.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:42:10.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/ts-0/data
23:42:10.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/ts-0/wal
23:42:10.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:42:10.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:42:10.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.34.1:0
23:42:10.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:42:10.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:42:10.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:42:10.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/ts-0/data/info.pb
23:42:10.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.34.1
23:42:10.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:42:10.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.34.62:41863,127.16.34.61:42279,127.16.34.60:36581
23:42:10.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:42:10.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:42:10.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:42:10.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:42:10.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:42:10.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.34.1
23:42:10.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/ts-0/logs
23:42:10.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:42:10.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:42:10.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
23:42:10.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:42:10.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:42:10.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:42:10.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:42:10.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 20:03:34 UTC on ccycloud.quasar-agiwfo.root.comops.site
23:42:10.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4576543
23:42:10.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:42:10.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:10.831458 16818 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:42:10.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:10.833459 16818 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:42:10.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:10.861001 16845 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:42:11.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:11.006417 16810 catalog_manager.cc:1259] Loaded cluster ID: 31c0efa5fe4a4c4da78972b2fb482efd
23:42:11.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:11.020838 16810 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 6fe54b56d7fb451e817d34984c697a86: loading cluster ID for follower catalog manager: success
23:42:11.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:11.063786 16810 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 6fe54b56d7fb451e817d34984c697a86: acquiring CA information for follower catalog manager: success
23:42:11.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:11.075086 16817 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6a9d90cd8a334f448572ee626a8b6c37 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "6a9d90cd8a334f448572ee626a8b6c37" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a9d90cd8a334f448572ee626a8b6c37" member_type: VOTER last_known_addr { host: "127.16.34.62" port: 41863 } } peers { permanent_uuid: "1d467dda25e54070a8644675f8b840cd" member_type: VOTER last_known_addr { host: "127.16.34.61" port: 42279 } } peers { permanent_uuid: "6fe54b56d7fb451e817d34984c697a86" member_type: VOTER last_known_addr { host: "127.16.34.60" port: 36581 } } }
23:42:11.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:11.075908 16817 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6a9d90cd8a334f448572ee626a8b6c37 [sys.catalog]: This master's current role is: LEADER
23:42:11.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:11.112828 16810 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 6fe54b56d7fb451e817d34984c697a86: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
23:42:13.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:12.251924 16844 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 16818
23:42:13.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:13.359244 16844 kernel_stack_watchdog.cc:198] Thread 16818 stuck at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:637 for 390ms:
23:42:13.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
23:42:13.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
23:42:13.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
23:42:13.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
23:42:13.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:10.863099 16846 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:42:13.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:13.364930 16818 thread.cc:636] GCE (cloud detector) Time spent creating pthread: real 2.506s user 0.830s sys 1.562s
23:42:13.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:13.365494 16818 thread.cc:602] GCE (cloud detector) Time spent starting thread: real 2.507s user 0.830s sys 1.562s
23:42:13.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:13.367889 16818 server_base.cc:1034] running on GCE node
23:42:13.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:13.369594 16855 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:42:13.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:13.371179 16818 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:42:13.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:13.382727 16818 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:42:13.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:13.393787 16818 hybrid_clock.cc:648] HybridClock initialized: now 1786318933393427 us; error 365 us; skew 500 ppm
23:42:13.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:13.396921 16818 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:42:13.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:13.424901 16818 webserver.cc:460] Webserver started at http://127.16.34.1:40393/ using document root <none> and password file <none>
23:42:13.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:13.427567 16818 fs_manager.cc:359] Metadata directory not provided
23:42:13.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:13.428079 16818 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:42:13.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:13.428869 16818 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
23:42:13.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:13.436770 16818 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/ts-0/data/instance:
23:42:13.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b98451df90304f4aa78abb5593673749"
23:42:13.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 23:42:13 on dist-test-slave-8w9v"
23:42:13.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:13.438716 16818 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/ts-0/wal/instance:
23:42:13.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b98451df90304f4aa78abb5593673749"
23:42:13.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 23:42:13 on dist-test-slave-8w9v"
23:42:13.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:13.458180 16818 fs_manager.cc:691] Time spent creating directory manager: real 0.018s user 0.007s sys 0.010s
23:42:13.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:13.471434 16863 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
23:42:13.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:13.472880 16818 fs_manager.cc:725] Time spent opening block manager: real 0.010s user 0.009s sys 0.001s
23:42:13.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:13.473402 16818 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/ts-0/data,/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/ts-0/wal
23:42:13.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b98451df90304f4aa78abb5593673749"
23:42:13.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 23:42:13 on dist-test-slave-8w9v"
23:42:13.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:13.474150 16818 fs_report.cc:383] FS layout report
23:42:13.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:42:13.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/ts-0/wal
23:42:13.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/ts-0/wal
23:42:13.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/ts-0/data/data
23:42:13.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:42:13.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:42:13.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:42:13.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:42:13.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:42:13.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:42:13.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:42:13.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:42:13.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:42:13.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:42:13.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:42:13.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:13.561151 16818 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
23:42:13.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:13.563370 16818 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:42:13.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:13.564114 16818 kserver.cc:163] Server-wide thread pool size limit: 3276
23:42:13.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:13.573663 16818 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
23:42:13.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:13.585765 16818 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
23:42:13.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:13.586236 16818 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.001s user 0.000s sys 0.001s
23:42:13.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:13.587379 16818 ts_tablet_manager.cc:610] Registered 0 tablets
23:42:13.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:13.587736 16818 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
23:42:14.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:14.135686 16818 rpc_server.cc:307] RPC server started. Bound to: 127.16.34.1:46209
23:42:14.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:14.135808 16983 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.34.1:46209 every 8 connection(s)
23:42:14.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:14.139101 16818 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/ts-0/data/info.pb
23:42:14.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:14.143965 16520 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskkf9L9h/build/tsan/bin/kudu as pid 16818
23:42:14.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:14.144716 16520 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/ts-0/wal/instance
23:42:14.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:14.159353 16520 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskkf9L9h/build/tsan/bin/kudu
23:42:14.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskkf9L9h/build/tsan/bin/kudu
23:42:14.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/ts-1/wal
23:42:14.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/ts-1/data
23:42:14.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:42:14.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:42:14.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:42:14.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:42:14.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:42:14.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:42:14.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/ts-1/logs
23:42:14.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/ts-1/data/info.pb
23:42:14.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:42:14.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:42:14.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:42:14.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:42:14.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:42:14.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:42:14.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:42:14.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:42:14.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:42:14.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:42:14.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.34.2:0
23:42:14.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.34.2
23:42:14.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.34.2
23:42:14.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:42:14.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.34.62:41863,127.16.34.61:42279,127.16.34.60:36581
23:42:14.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.34.20:32993
23:42:14.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:42:14.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:42:14.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:42:14.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:14.238073 16985 heartbeater.cc:347] Connected to a master server at 127.16.34.61:42279
23:42:14.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:14.238582 16985 heartbeater.cc:464] Registering TS with master...
23:42:14.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:14.239949 16985 heartbeater.cc:511] Master 127.16.34.61:42279 requested a full tablet report, sending...
23:42:14.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:14.246048 16649 ts_manager.cc:194] Registered new tserver with Master: b98451df90304f4aa78abb5593673749 (127.16.34.1:46209)
23:42:14.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:14.252117 16984 heartbeater.cc:347] Connected to a master server at 127.16.34.60:36581
23:42:14.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:14.252543 16984 heartbeater.cc:464] Registering TS with master...
23:42:14.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:14.253454 16984 heartbeater.cc:511] Master 127.16.34.60:36581 requested a full tablet report, sending...
23:42:14.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:14.257772 16731 ts_manager.cc:194] Registered new tserver with Master: b98451df90304f4aa78abb5593673749 (127.16.34.1:46209)
23:42:14.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:14.265098 16989 heartbeater.cc:347] Connected to a master server at 127.16.34.62:41863
23:42:14.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:14.265462 16989 heartbeater.cc:464] Registering TS with master...
23:42:14.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:14.266232 16989 heartbeater.cc:511] Master 127.16.34.62:41863 requested a full tablet report, sending...
23:42:14.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:14.268409 16568 ts_manager.cc:194] Registered new tserver with Master: b98451df90304f4aa78abb5593673749 (127.16.34.1:46209)
23:42:14.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:14.270871 16568 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.16.34.1:48329
23:42:14.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:14.544281 16992 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:42:14.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:14.544826 16992 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:42:14.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:14.545396 16992 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:42:14.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:14.576067 16992 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:42:14.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:14.577041 16992 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.34.2
23:42:14.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:14.611184 16992 tablet_server_runner.cc:78] Tablet server non-default flags:
23:42:14.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:42:14.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.34.20:32993
23:42:14.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:42:14.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:42:14.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/ts-1/data
23:42:14.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/ts-1/wal
23:42:14.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:42:14.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:42:14.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.34.2:0
23:42:14.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:42:14.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:42:14.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:42:14.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/ts-1/data/info.pb
23:42:14.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.34.2
23:42:14.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:42:14.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.34.62:41863,127.16.34.61:42279,127.16.34.60:36581
23:42:14.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:42:14.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:42:14.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:42:14.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:42:14.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:42:14.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.34.2
23:42:14.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/ts-1/logs
23:42:14.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:42:14.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:42:14.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
23:42:14.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:42:14.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:42:14.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:42:14.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:42:14.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 20:03:34 UTC on ccycloud.quasar-agiwfo.root.comops.site
23:42:14.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4576543
23:42:14.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:42:14.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:14.612823 16992 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:42:14.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:14.614373 16992 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:42:14.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:14.634179 17003 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:42:15.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:15.274539 16989 heartbeater.cc:503] Master 127.16.34.62:41863 was elected leader, sending a full tablet report...
23:42:15.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:14.635732 17004 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:42:15.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:14.636188 17006 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:42:15.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:14.636619 16992 server_base.cc:1034] running on GCE node
23:42:15.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:15.828764 16992 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:42:15.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:15.831238 16992 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:42:15.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:15.832609 16992 hybrid_clock.cc:648] HybridClock initialized: now 1786318935832577 us; error 41 us; skew 500 ppm
23:42:15.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:15.833361 16992 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:42:15.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:15.843101 16992 webserver.cc:460] Webserver started at http://127.16.34.2:36679/ using document root <none> and password file <none>
23:42:15.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:15.844074 16992 fs_manager.cc:359] Metadata directory not provided
23:42:15.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:15.844372 16992 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:42:15.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:15.844918 16992 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
23:42:15.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:15.849774 16992 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/ts-1/data/instance:
23:42:15.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fe579bcace6b470987ddb19e6a85da78"
23:42:15.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 23:42:15 on dist-test-slave-8w9v"
23:42:15.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:15.850986 16992 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/ts-1/wal/instance:
23:42:15.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fe579bcace6b470987ddb19e6a85da78"
23:42:15.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 23:42:15 on dist-test-slave-8w9v"
23:42:15.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:15.857788 16992 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.005s sys 0.003s
23:42:15.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:15.863095 17013 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
23:42:15.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:15.864001 16992 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
23:42:15.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:15.864377 16992 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/ts-1/data,/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/ts-1/wal
23:42:15.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fe579bcace6b470987ddb19e6a85da78"
23:42:15.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 23:42:15 on dist-test-slave-8w9v"
23:42:15.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:15.864717 16992 fs_report.cc:383] FS layout report
23:42:15.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:42:15.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/ts-1/wal
23:42:15.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/ts-1/wal
23:42:15.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/ts-1/data/data
23:42:15.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:42:15.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:42:15.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:42:15.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:42:15.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:42:15.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:42:15.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:42:15.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:42:15.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:42:15.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:42:15.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:42:15.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:15.909866 16992 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
23:42:15.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:15.911593 16992 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:42:15.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:15.912185 16992 kserver.cc:163] Server-wide thread pool size limit: 3276
23:42:15.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:15.914685 16992 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
23:42:15.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:15.918901 16992 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
23:42:15.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:15.919219 16992 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
23:42:15.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:15.919554 16992 ts_tablet_manager.cc:610] Registered 0 tablets
23:42:15.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:15.919754 16992 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
23:42:16.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:16.076892 16992 rpc_server.cc:307] RPC server started. Bound to: 127.16.34.2:34411
23:42:16.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:16.077030 17132 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.34.2:34411 every 8 connection(s)
23:42:16.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:16.079468 16992 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/ts-1/data/info.pb
23:42:16.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:16.087215 16520 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskkf9L9h/build/tsan/bin/kudu as pid 16992
23:42:16.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:16.087658 16520 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/ts-1/wal/instance
23:42:16.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:16.097666 16520 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskkf9L9h/build/tsan/bin/kudu
23:42:16.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskkf9L9h/build/tsan/bin/kudu
23:42:16.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/ts-2/wal
23:42:16.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/ts-2/data
23:42:16.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:42:16.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:42:16.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:42:16.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:42:16.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:42:16.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:42:16.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/ts-2/logs
23:42:16.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/ts-2/data/info.pb
23:42:16.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:42:16.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:42:16.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:42:16.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:42:16.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:42:16.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:42:16.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:42:16.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:42:16.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:42:16.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:42:16.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.34.3:0
23:42:16.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.34.3
23:42:16.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.34.3
23:42:16.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:42:16.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.34.62:41863,127.16.34.61:42279,127.16.34.60:36581
23:42:16.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.34.20:32993
23:42:16.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:42:16.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:42:16.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:42:16.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:16.160585 17134 heartbeater.cc:347] Connected to a master server at 127.16.34.61:42279
23:42:16.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:16.161077 17134 heartbeater.cc:464] Registering TS with master...
23:42:16.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:16.162358 17134 heartbeater.cc:511] Master 127.16.34.61:42279 requested a full tablet report, sending...
23:42:16.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:16.163368 17133 heartbeater.cc:347] Connected to a master server at 127.16.34.60:36581
23:42:16.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:16.163753 17133 heartbeater.cc:464] Registering TS with master...
23:42:16.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:16.164911 17133 heartbeater.cc:511] Master 127.16.34.60:36581 requested a full tablet report, sending...
23:42:16.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:16.165560 16649 ts_manager.cc:194] Registered new tserver with Master: fe579bcace6b470987ddb19e6a85da78 (127.16.34.2:34411)
23:42:16.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:16.167582 16731 ts_manager.cc:194] Registered new tserver with Master: fe579bcace6b470987ddb19e6a85da78 (127.16.34.2:34411)
23:42:16.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:16.169287 17135 heartbeater.cc:347] Connected to a master server at 127.16.34.62:41863
23:42:16.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:16.169553 17135 heartbeater.cc:464] Registering TS with master...
23:42:16.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:16.170223 17135 heartbeater.cc:511] Master 127.16.34.62:41863 requested a full tablet report, sending...
23:42:16.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:16.172008 16568 ts_manager.cc:194] Registered new tserver with Master: fe579bcace6b470987ddb19e6a85da78 (127.16.34.2:34411)
23:42:16.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:16.173592 16568 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.16.34.2:56291
23:42:16.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:16.468815 17143 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:42:16.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:16.469247 17143 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:42:16.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:16.469655 17143 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:42:16.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:16.499136 17143 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:42:16.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:16.500137 17143 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.34.3
23:42:16.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:16.533061 17143 tablet_server_runner.cc:78] Tablet server non-default flags:
23:42:16.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:42:16.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.34.20:32993
23:42:16.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:42:16.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:42:16.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/ts-2/data
23:42:16.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/ts-2/wal
23:42:16.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:42:16.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:42:16.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.34.3:0
23:42:16.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:42:16.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:42:16.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:42:16.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/ts-2/data/info.pb
23:42:16.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.34.3
23:42:16.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:42:16.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.34.62:41863,127.16.34.61:42279,127.16.34.60:36581
23:42:16.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:42:16.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:42:16.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:42:16.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:42:16.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:42:16.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.34.3
23:42:16.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/ts-2/logs
23:42:16.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:42:16.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:42:16.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
23:42:16.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:42:16.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:42:16.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:42:16.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:42:16.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 20:03:34 UTC on ccycloud.quasar-agiwfo.root.comops.site
23:42:16.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4576543
23:42:16.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:42:16.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:16.534200 17143 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:42:16.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:16.535980 17143 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:42:16.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:16.557169 17152 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:42:17.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:17.176874 17135 heartbeater.cc:503] Master 127.16.34.62:41863 was elected leader, sending a full tablet report...
23:42:17.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:16.557176 17154 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:42:17.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:16.558178 17151 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:42:17.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:17.736384 17153 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
23:42:17.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:17.736505 17143 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
23:42:17.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:17.739709 17143 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:42:17.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:17.742290 17143 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:42:17.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:17.743652 17143 hybrid_clock.cc:648] HybridClock initialized: now 1786318937743623 us; error 36 us; skew 500 ppm
23:42:17.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:17.744825 17143 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:42:17.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:17.753235 17143 webserver.cc:460] Webserver started at http://127.16.34.3:42765/ using document root <none> and password file <none>
23:42:17.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:17.754045 17143 fs_manager.cc:359] Metadata directory not provided
23:42:17.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:17.754217 17143 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:42:17.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:17.754585 17143 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
23:42:17.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:17.759271 17143 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/ts-2/data/instance:
23:42:17.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5f595e179d784ab3951720738f86268a"
23:42:17.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 23:42:17 on dist-test-slave-8w9v"
23:42:17.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:17.760552 17143 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/ts-2/wal/instance:
23:42:17.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5f595e179d784ab3951720738f86268a"
23:42:17.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 23:42:17 on dist-test-slave-8w9v"
23:42:17.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:17.767607 17143 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.006s sys 0.000s
23:42:17.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:17.772878 17162 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
23:42:17.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:17.773842 17143 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.002s
23:42:17.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:17.774143 17143 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/ts-2/data,/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/ts-2/wal
23:42:17.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5f595e179d784ab3951720738f86268a"
23:42:17.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 23:42:17 on dist-test-slave-8w9v"
23:42:17.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:17.774464 17143 fs_report.cc:383] FS layout report
23:42:17.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:42:17.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/ts-2/wal
23:42:17.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/ts-2/wal
23:42:17.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/ts-2/data/data
23:42:17.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:42:17.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:42:17.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:42:17.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:42:17.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:42:17.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:42:17.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:42:17.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:42:17.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:42:17.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:42:17.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:42:17.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:17.818670 17143 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
23:42:17.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:17.820261 17143 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:42:17.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:17.820662 17143 kserver.cc:163] Server-wide thread pool size limit: 3276
23:42:17.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:17.823115 17143 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
23:42:17.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:17.827093 17143 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
23:42:17.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:17.827296 17143 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
23:42:17.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:17.827491 17143 ts_tablet_manager.cc:610] Registered 0 tablets
23:42:17.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:17.827617 17143 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
23:42:17.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:17.983732 17143 rpc_server.cc:307] RPC server started. Bound to: 127.16.34.3:43619
23:42:17.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:17.983846 17282 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.34.3:43619 every 8 connection(s)
23:42:17.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:17.986519 17143 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/ts-2/data/info.pb
23:42:17.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:17.989071 16520 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskkf9L9h/build/tsan/bin/kudu as pid 17143
23:42:17.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:17.989614 16520 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster4048038085134533336/ts-2/wal/instance
23:42:18.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:18.034462 17284 heartbeater.cc:347] Connected to a master server at 127.16.34.61:42279
23:42:18.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:18.034971 17284 heartbeater.cc:464] Registering TS with master...
23:42:18.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:18.036248 17283 heartbeater.cc:347] Connected to a master server at 127.16.34.60:36581
23:42:18.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:18.036303 17284 heartbeater.cc:511] Master 127.16.34.61:42279 requested a full tablet report, sending...
23:42:18.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:18.036604 17283 heartbeater.cc:464] Registering TS with master...
23:42:18.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:18.036914 17285 heartbeater.cc:347] Connected to a master server at 127.16.34.62:41863
23:42:18.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:18.037233 17285 heartbeater.cc:464] Registering TS with master...
23:42:18.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:18.037288 17283 heartbeater.cc:511] Master 127.16.34.60:36581 requested a full tablet report, sending...
23:42:18.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:18.038030 17285 heartbeater.cc:511] Master 127.16.34.62:41863 requested a full tablet report, sending...
23:42:18.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:18.039981 16731 ts_manager.cc:194] Registered new tserver with Master: 5f595e179d784ab3951720738f86268a (127.16.34.3:43619)
23:42:18.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:18.039928 16649 ts_manager.cc:194] Registered new tserver with Master: 5f595e179d784ab3951720738f86268a (127.16.34.3:43619)
23:42:18.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:18.040220 16568 ts_manager.cc:194] Registered new tserver with Master: 5f595e179d784ab3951720738f86268a (127.16.34.3:43619)
23:42:18.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:18.042160 16568 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.16.34.3:55603
23:42:18.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:18.050382 16520 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
23:42:18.060 [DEBUG - main] (MiniKuduCluster.java:195) Response:
23:42:18.062 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
23:42:18.072 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.16.34.62"
port: 41863
}
bound_http_address {
host: "127.16.34.62"
port: 34005
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.16.34.61"
port: 42279
}
bound_http_address {
host: "127.16.34.61"
port: 41267
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.16.34.60"
port: 36581
}
bound_http_address {
host: "127.16.34.60"
port: 46487
}
}
}
23:42:18.078 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
23:42:18.081 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.16.34.1"
port: 46209
}
bound_http_address {
host: "127.16.34.1"
port: 40393
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.16.34.2"
port: 34411
}
bound_http_address {
host: "127.16.34.2"
port: 36679
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.16.34.3"
port: 43619
}
bound_http_address {
host: "127.16.34.3"
port: 42765
}
}
}
23:42:18.081 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
23:42:18.265 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
23:42:18.372 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskkf9L9h/test-tmp
23:42:18.372 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
23:42:18.373 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskkf9L9h/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
23:42:18.378 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
23:42:18.702 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
23:42:18.702 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
23:42:18.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:18.706867 17295 mini_oidc.cc:152] Starting JWKS server
23:42:18.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:18.711229 17295 webserver.cc:460] Webserver started at http://127.0.0.1:41091/ using document root <none> and password file <none>
23:42:18.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:18.711737 17295 mini_oidc.cc:189] Starting OIDC Discovery server
23:42:18.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:18.713814 17295 webserver.cc:460] Webserver started at http://127.0.0.1:43429/ using document root <none> and password file <none>
23:42:18.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
23:42:18.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/krb5kdc/principal' for realm 'KRBTEST.COM',
23:42:18.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
23:42:18.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 23:42:18 dist-test-slave-8w9v krb5kdc[17308](info): setting up network...
23:42:18.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
23:42:18.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 23:42:18 dist-test-slave-8w9v krb5kdc[17308](info): set up 2 sockets
23:42:18.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 23:42:18 dist-test-slave-8w9v krb5kdc[17308](info): commencing operation
23:42:18.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
23:42:19.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:19.045883 17285 heartbeater.cc:503] Master 127.16.34.62:41863 was elected leader, sending a full tablet report...
23:42:19.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:19.913035 17295 mini_kdc.cc:121] Time spent starting KDC: real 1.169s user 0.007s sys 0.013s
23:42:19.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
23:42:19.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
23:42:19.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
23:42:19.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
23:42:19.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
23:42:19.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
23:42:19.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
23:42:20.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
23:42:20.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
23:42:20.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
23:42:20.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/krb5kdc/test-user.keytab.
23:42:20.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/krb5kdc/test-user.keytab.
23:42:20.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 23:42:20 dist-test-slave-8w9v krb5kdc[17308](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786318940, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
23:42:20.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
23:42:20.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-09T23:42:20Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
23:42:20.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-09T23:42:20Z Disabled control of system clock
23:42:20.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.16.227.254@KRBTEST.COM; defaulting to no policy
23:42:20.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
23:42:20.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.16.227.254@KRBTEST.COM" created.
23:42:20.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
23:42:20.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.16.227.254 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/krb5kdc/kudu_127.16.227.254.keytab.
23:42:20.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.16.227.254 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/krb5kdc/kudu_127.16.227.254.keytab.
23:42:20.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:20.129276 17295 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskkf9L9h/build/tsan/bin/kudu
23:42:20.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskkf9L9h/build/tsan/bin/kudu
23:42:20.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-0/wal
23:42:20.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-0/data
23:42:20.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:42:20.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:42:20.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:42:20.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:42:20.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:42:20.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:42:20.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-0/logs
23:42:20.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-0/data/info.pb
23:42:20.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:42:20.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:42:20.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:42:20.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:42:20.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:42:20.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:42:20.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:42:20.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:42:20.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
23:42:20.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:42:20.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:42:20.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:42:20.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.227.254:42395
23:42:20.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.227.254
23:42:20.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:42:20.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/krb5kdc/kudu_127.16.227.254.keytab
23:42:20.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.16.227.254
23:42:20.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
23:42:20.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
23:42:20.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
23:42:20.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.227.212:35697
23:42:20.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:42:20.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:42:20.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:42:20.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:42:20.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.227.254:42395,127.16.227.253:34331,127.16.227.252:34209
23:42:20.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:42:20.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41091/jwks.json
23:42:20.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
23:42:20.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:20.561192 17322 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:42:20.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:20.561846 17322 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:42:20.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:20.562424 17322 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:42:20.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:20.599709 17322 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
23:42:20.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:20.599992 17322 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
23:42:20.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:20.600461 17322 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:42:20.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:20.600783 17322 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
23:42:20.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:20.601115 17322 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
23:42:20.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:20.601441 17322 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
23:42:20.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:20.601665 17322 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41091/jwks.json
23:42:20.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:20.638703 17322 master_runner.cc:386] Master server non-default flags:
23:42:20.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:42:20.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.227.212:35697
23:42:20.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:42:20.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:42:20.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-0/data
23:42:20.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-0/wal
23:42:20.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
23:42:20.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:42:20.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.227.254:42395,127.16.227.253:34331,127.16.227.252:34209
23:42:20.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/krb5kdc/kudu_127.16.227.254.keytab
23:42:20.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.16.227.254
23:42:20.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:42:20.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:42:20.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:42:20.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.227.254:42395
23:42:20.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:42:20.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:42:20.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:42:20.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41091/jwks.json
23:42:20.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
23:42:20.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
23:42:20.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
23:42:20.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:42:20.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:42:20.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-0/data/info.pb
23:42:20.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.227.254
23:42:20.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:42:20.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:42:20.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:42:20.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:42:20.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:42:20.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:42:20.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-0/logs
23:42:20.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:42:20.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:42:20.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
23:42:20.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
23:42:20.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:42:20.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:42:20.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:42:20.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 20:03:34 UTC on ccycloud.quasar-agiwfo.root.comops.site
23:42:20.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4576543
23:42:20.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:42:20.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:20.640321 17322 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:42:20.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:20.641774 17322 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:42:20.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:20.688169 17329 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:42:22.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:20.689254 17332 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:42:22.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:20.688238 17328 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:42:22.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:22.131500 17331 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1440 milliseconds
23:42:22.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:22.131649 17322 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
23:42:22.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:22.133167 17322 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:42:22.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:22.139499 17322 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:42:22.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:22.141137 17322 hybrid_clock.cc:648] HybridClock initialized: now 1786318942141062 us; error 89 us; skew 500 ppm
23:42:22.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:22.142299 17322 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:42:22.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 23:42:22 dist-test-slave-8w9v krb5kdc[17308](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786318942, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.227.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
23:42:22.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:22.159521 17322 init.cc:377] Logged in from keytab as kudu/127.16.227.254@KRBTEST.COM (short username kudu)
23:42:22.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:22.169205 17322 webserver.cc:460] Webserver started at http://127.16.227.254:41533/ using document root <none> and password file <none>
23:42:22.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:22.170102 17322 fs_manager.cc:359] Metadata directory not provided
23:42:22.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:22.170286 17322 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:42:22.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:22.170675 17322 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
23:42:22.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:22.175429 17322 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-0/data/instance:
23:42:22.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aa1cf38e72e643c39ce903ca13ecdd81"
23:42:22.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 23:42:22 on dist-test-slave-8w9v"
23:42:22.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:22.176604 17322 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-0/wal/instance:
23:42:22.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aa1cf38e72e643c39ce903ca13ecdd81"
23:42:22.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 23:42:22 on dist-test-slave-8w9v"
23:42:22.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:22.187130 17322 fs_manager.cc:691] Time spent creating directory manager: real 0.010s user 0.004s sys 0.004s
23:42:22.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:22.199481 17342 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
23:42:22.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:22.201082 17322 fs_manager.cc:725] Time spent opening block manager: real 0.009s user 0.001s sys 0.005s
23:42:22.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:22.201412 17322 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-0/data,/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-0/wal
23:42:22.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aa1cf38e72e643c39ce903ca13ecdd81"
23:42:22.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 23:42:22 on dist-test-slave-8w9v"
23:42:22.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:22.201752 17322 fs_report.cc:383] FS layout report
23:42:22.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:42:22.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-0/wal
23:42:22.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-0/wal
23:42:22.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-0/data/data
23:42:22.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:42:22.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:42:22.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:42:22.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:42:22.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:42:22.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:42:22.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:42:22.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:42:22.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:42:22.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:42:22.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:42:22.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:22.257416 17322 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
23:42:22.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:22.264938 17322 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:42:22.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:22.265543 17322 kserver.cc:163] Server-wide thread pool size limit: 3276
23:42:22.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:22.500386 17322 rpc_server.cc:307] RPC server started. Bound to: 127.16.227.254:42395
23:42:22.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:22.500640 17397 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.227.254:42395 every 8 connection(s)
23:42:22.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:22.503193 17322 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-0/data/info.pb
23:42:22.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:22.513476 17295 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskkf9L9h/build/tsan/bin/kudu as pid 17322
23:42:22.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:22.514068 17295 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-0/wal/instance
23:42:22.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:22.514611 17398 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
23:42:22.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.16.227.253@KRBTEST.COM; defaulting to no policy
23:42:22.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:22.537578 17398 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.227.254" port: 42395 } has no permanent_uuid. Determining permanent_uuid...
23:42:22.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
23:42:22.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.16.227.253@KRBTEST.COM" created.
23:42:22.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
23:42:22.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.16.227.253 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/krb5kdc/kudu_127.16.227.253.keytab.
23:42:22.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.16.227.253 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/krb5kdc/kudu_127.16.227.253.keytab.
23:42:22.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:22.577085 17295 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskkf9L9h/build/tsan/bin/kudu
23:42:22.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskkf9L9h/build/tsan/bin/kudu
23:42:22.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-1/wal
23:42:22.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-1/data
23:42:22.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:42:22.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:42:22.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:42:22.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:42:22.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:42:22.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:42:22.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-1/logs
23:42:22.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-1/data/info.pb
23:42:22.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:42:22.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:42:22.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:42:22.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:42:22.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:42:22.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:42:22.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:42:22.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:42:22.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
23:42:22.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:42:22.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:42:22.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:42:22.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.227.253:34331
23:42:22.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.227.253
23:42:22.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:42:22.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/krb5kdc/kudu_127.16.227.253.keytab
23:42:22.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.16.227.253
23:42:22.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
23:42:22.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
23:42:22.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
23:42:22.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.227.212:35697
23:42:22.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:42:22.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:42:22.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:42:22.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:42:22.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.227.254:42395,127.16.227.253:34331,127.16.227.252:34209
23:42:22.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:42:22.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41091/jwks.json
23:42:22.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
23:42:22.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 23:42:22 dist-test-slave-8w9v krb5kdc[17308](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786318942, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.227.254@KRBTEST.COM for kudu/127.16.227.254@KRBTEST.COM
23:42:22.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:22.601794 17398 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.227.253" port: 34331 } has no permanent_uuid. Determining permanent_uuid...
23:42:22.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:22.606621 17347 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.227.253:34331: connect: Connection refused (error 111)
23:42:22.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:22.617375 17398 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.227.253:34331: Network error: Client connection negotiation failed: client connection to 127.16.227.253:34331: connect: Connection refused (error 111)
23:42:22.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:22.667073 17398 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.227.253" port: 34331 } attempt: 1
23:42:22.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:22.672742 17398 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.227.253:34331: Network error: Client connection negotiation failed: client connection to 127.16.227.253:34331: connect: Connection refused (error 111)
23:42:22.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:22.741420 17398 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.227.253" port: 34331 } attempt: 2
23:42:22.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:22.747527 17398 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.227.253:34331: Network error: Client connection negotiation failed: client connection to 127.16.227.253:34331: connect: Connection refused (error 111)
23:42:22.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:22.838929 17398 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.227.253" port: 34331 } attempt: 3
23:42:22.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:22.844321 17398 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.227.253:34331: Network error: Client connection negotiation failed: client connection to 127.16.227.253:34331: connect: Connection refused (error 111)
23:42:22.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:22.987818 17398 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.227.253" port: 34331 } attempt: 4
23:42:22.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:22.993157 17398 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.227.253:34331: Network error: Client connection negotiation failed: client connection to 127.16.227.253:34331: connect: Connection refused (error 111)
23:42:23.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:23.079900 17405 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:42:23.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:23.080739 17405 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:42:23.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:23.081483 17405 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:42:23.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:23.114418 17405 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
23:42:23.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:23.114681 17405 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
23:42:23.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:23.115105 17405 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:42:23.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:23.115419 17405 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
23:42:23.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:23.115710 17405 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
23:42:23.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:23.116019 17405 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
23:42:23.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:23.116261 17405 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41091/jwks.json
23:42:23.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:23.153247 17405 master_runner.cc:386] Master server non-default flags:
23:42:23.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:42:23.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.227.212:35697
23:42:23.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:42:23.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:42:23.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-1/data
23:42:23.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-1/wal
23:42:23.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
23:42:23.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:42:23.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.227.254:42395,127.16.227.253:34331,127.16.227.252:34209
23:42:23.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/krb5kdc/kudu_127.16.227.253.keytab
23:42:23.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.16.227.253
23:42:23.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:42:23.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:42:23.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:42:23.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.227.253:34331
23:42:23.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:42:23.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:42:23.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:42:23.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41091/jwks.json
23:42:23.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
23:42:23.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
23:42:23.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
23:42:23.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:42:23.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:42:23.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-1/data/info.pb
23:42:23.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.227.253
23:42:23.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:42:23.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:42:23.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:42:23.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:42:23.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:42:23.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:42:23.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-1/logs
23:42:23.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:42:23.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:42:23.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
23:42:23.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
23:42:23.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:42:23.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:42:23.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:42:23.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 20:03:34 UTC on ccycloud.quasar-agiwfo.root.comops.site
23:42:23.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4576543
23:42:23.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:42:23.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:23.154734 17405 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:42:23.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:23.156172 17405 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:42:23.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:23.188697 17416 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:42:23.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:23.292680 17398 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.227.253" port: 34331 } attempt: 5
23:42:23.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:23.299192 17398 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.227.253:34331: Network error: Client connection negotiation failed: client connection to 127.16.227.253:34331: connect: Connection refused (error 111)
23:42:23.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:23.853127 17398 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.227.253" port: 34331 } attempt: 6
23:42:23.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:23.865082 17398 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.227.253:34331: Network error: Client connection negotiation failed: client connection to 127.16.227.253:34331: connect: Connection refused (error 111)
23:42:24.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:23.190255 17415 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:42:24.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:23.189957 17418 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:42:24.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:23.201097 17405 server_base.cc:1034] running on GCE node
23:42:24.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:24.564327 17405 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:42:24.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:24.568840 17405 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:42:24.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:24.570271 17405 hybrid_clock.cc:648] HybridClock initialized: now 1786318944570232 us; error 64 us; skew 500 ppm
23:42:24.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:24.571187 17405 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:42:24.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 23:42:24 dist-test-slave-8w9v krb5kdc[17308](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786318944, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.227.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
23:42:24.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:24.579600 17405 init.cc:377] Logged in from keytab as kudu/127.16.227.253@KRBTEST.COM (short username kudu)
23:42:24.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:24.591264 17405 webserver.cc:460] Webserver started at http://127.16.227.253:36583/ using document root <none> and password file <none>
23:42:24.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:24.592283 17405 fs_manager.cc:359] Metadata directory not provided
23:42:24.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:24.592542 17405 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:42:24.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:24.593011 17405 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
23:42:24.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:24.598289 17405 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-1/data/instance:
23:42:24.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8fdb81d4101b4b3bb72f6e2f36f173c5"
23:42:24.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 23:42:24 on dist-test-slave-8w9v"
23:42:24.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:24.599514 17405 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-1/wal/instance:
23:42:24.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8fdb81d4101b4b3bb72f6e2f36f173c5"
23:42:24.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 23:42:24 on dist-test-slave-8w9v"
23:42:24.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:24.610173 17405 fs_manager.cc:691] Time spent creating directory manager: real 0.010s user 0.012s sys 0.000s
23:42:24.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:24.618853 17432 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
23:42:24.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:24.620172 17405 fs_manager.cc:725] Time spent opening block manager: real 0.006s user 0.002s sys 0.004s
23:42:24.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:24.620493 17405 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-1/data,/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-1/wal
23:42:24.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8fdb81d4101b4b3bb72f6e2f36f173c5"
23:42:24.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 23:42:24 on dist-test-slave-8w9v"
23:42:24.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:24.620806 17405 fs_report.cc:383] FS layout report
23:42:24.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:42:24.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-1/wal
23:42:24.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-1/wal
23:42:24.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-1/data/data
23:42:24.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:42:24.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:42:24.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:42:24.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:42:24.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:42:24.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:42:24.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:42:24.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:42:24.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:42:24.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:42:24.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:42:24.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:24.637051 17394 debug-util.cc:398] Leaking SignalData structure 0x7b08000734a0 after lost signal to thread 17323
23:42:24.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:24.637876 17394 debug-util.cc:398] Leaking SignalData structure 0x7b080007e2c0 after lost signal to thread 17397
23:42:24.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:24.676301 17405 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
23:42:24.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:24.684326 17405 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:42:24.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:24.684899 17405 kserver.cc:163] Server-wide thread pool size limit: 3276
23:42:24.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:24.869051 17405 rpc_server.cc:307] RPC server started. Bound to: 127.16.227.253:34331
23:42:24.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:24.869496 17489 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.227.253:34331 every 8 connection(s)
23:42:24.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:24.872314 17405 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-1/data/info.pb
23:42:24.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:24.875196 17295 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskkf9L9h/build/tsan/bin/kudu as pid 17405
23:42:24.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:24.875871 17295 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-1/wal/instance
23:42:24.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:24.892000 17490 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
23:42:24.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.16.227.252@KRBTEST.COM; defaulting to no policy
23:42:24.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
23:42:24.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.16.227.252@KRBTEST.COM" created.
23:42:24.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:24.919338 17490 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.227.254" port: 42395 } has no permanent_uuid. Determining permanent_uuid...
23:42:24.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:24.925808 17398 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.227.253" port: 34331 } attempt: 7
23:42:24.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
23:42:24.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.16.227.252 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/krb5kdc/kudu_127.16.227.252.keytab.
23:42:24.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.16.227.252 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/krb5kdc/kudu_127.16.227.252.keytab.
23:42:24.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:24.961882 17295 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskkf9L9h/build/tsan/bin/kudu
23:42:24.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskkf9L9h/build/tsan/bin/kudu
23:42:24.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-2/wal
23:42:24.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-2/data
23:42:24.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:42:24.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:42:24.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:42:24.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:42:24.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:42:24.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:42:24.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-2/logs
23:42:24.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-2/data/info.pb
23:42:24.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:42:24.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:42:24.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:42:24.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:42:24.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:42:24.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:42:24.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:42:24.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:42:24.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
23:42:24.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:42:24.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:42:24.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:42:24.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.227.252:34209
23:42:24.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.227.252
23:42:24.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:42:24.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/krb5kdc/kudu_127.16.227.252.keytab
23:42:24.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.16.227.252
23:42:24.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
23:42:24.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
23:42:24.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
23:42:24.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.227.212:35697
23:42:24.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:42:24.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:42:24.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:42:24.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:42:24.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.227.254:42395,127.16.227.253:34331,127.16.227.252:34209
23:42:24.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:42:24.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41091/jwks.json
23:42:24.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
23:42:25.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 23:42:25 dist-test-slave-8w9v krb5kdc[17308](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786318944, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.227.253@KRBTEST.COM for kudu/127.16.227.254@KRBTEST.COM
23:42:25.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 23:42:25 dist-test-slave-8w9v krb5kdc[17308](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786318942, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.227.254@KRBTEST.COM for kudu/127.16.227.253@KRBTEST.COM
23:42:25.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:25.039361 17490 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.227.253" port: 34331 } has no permanent_uuid. Determining permanent_uuid...
23:42:25.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:25.043552 17398 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.227.252" port: 34209 } has no permanent_uuid. Determining permanent_uuid...
23:42:25.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:25.059962 17398 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.227.252:34209: Network error: Client connection negotiation failed: client connection to 127.16.227.252:34209: connect: Connection refused (error 111)
23:42:25.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 23:42:25 dist-test-slave-8w9v krb5kdc[17308](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786318944, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.227.253@KRBTEST.COM for kudu/127.16.227.253@KRBTEST.COM
23:42:25.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:25.085650 17490 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.227.252" port: 34209 } has no permanent_uuid. Determining permanent_uuid...
23:42:25.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:25.098780 17435 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.227.252:34209: connect: Connection refused (error 111)
23:42:25.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:25.118543 17398 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.227.252" port: 34209 } attempt: 1
23:42:25.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:25.119181 17490 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.227.252:34209: Network error: Client connection negotiation failed: client connection to 127.16.227.252:34209: connect: Connection refused (error 111)
23:42:25.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:25.125231 17398 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.227.252:34209: Network error: Client connection negotiation failed: client connection to 127.16.227.252:34209: connect: Connection refused (error 111)
23:42:25.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:25.168923 17490 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.227.252" port: 34209 } attempt: 1
23:42:25.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:25.176267 17490 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.227.252:34209: Network error: Client connection negotiation failed: client connection to 127.16.227.252:34209: connect: Connection refused (error 111)
23:42:25.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:25.206772 17398 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.227.252" port: 34209 } attempt: 2
23:42:25.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:25.225361 17398 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.227.252:34209: Network error: Client connection negotiation failed: client connection to 127.16.227.252:34209: connect: Connection refused (error 111)
23:42:25.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:25.244757 17490 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.227.252" port: 34209 } attempt: 2
23:42:25.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:25.253819 17490 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.227.252:34209: Network error: Client connection negotiation failed: client connection to 127.16.227.252:34209: connect: Connection refused (error 111)
23:42:25.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:25.310966 17398 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.227.252" port: 34209 } attempt: 3
23:42:25.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:25.323108 17398 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.227.252:34209: Network error: Client connection negotiation failed: client connection to 127.16.227.252:34209: connect: Connection refused (error 111)
23:42:25.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:25.345553 17490 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.227.252" port: 34209 } attempt: 3
23:42:25.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:25.353027 17490 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.227.252:34209: Network error: Client connection negotiation failed: client connection to 127.16.227.252:34209: connect: Connection refused (error 111)
23:42:25.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:25.463668 17398 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.227.252" port: 34209 } attempt: 4
23:42:25.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:25.470923 17398 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.227.252:34209: Network error: Client connection negotiation failed: client connection to 127.16.227.252:34209: connect: Connection refused (error 111)
23:42:25.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:25.496673 17490 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.227.252" port: 34209 } attempt: 4
23:42:25.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:25.504624 17490 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.227.252:34209: Network error: Client connection negotiation failed: client connection to 127.16.227.252:34209: connect: Connection refused (error 111)
23:42:25.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:25.649510 17498 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:42:25.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:25.650540 17498 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:42:25.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:25.651343 17498 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:42:25.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:25.692600 17498 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
23:42:25.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:25.692906 17498 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
23:42:25.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:25.693218 17498 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:42:25.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:25.693418 17498 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
23:42:25.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:25.693596 17498 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
23:42:25.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:25.693801 17498 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
23:42:25.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:25.693940 17498 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41091/jwks.json
23:42:25.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:25.754514 17398 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.227.252" port: 34209 } attempt: 5
23:42:25.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:25.760391 17398 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.227.252:34209: Network error: Client connection negotiation failed: client connection to 127.16.227.252:34209: connect: Connection refused (error 111)
23:42:25.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:25.770124 17498 master_runner.cc:386] Master server non-default flags:
23:42:25.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:42:25.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.227.212:35697
23:42:25.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:42:25.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:42:25.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-2/data
23:42:25.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-2/wal
23:42:25.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
23:42:25.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:42:25.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.227.254:42395,127.16.227.253:34331,127.16.227.252:34209
23:42:25.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/krb5kdc/kudu_127.16.227.252.keytab
23:42:25.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.16.227.252
23:42:25.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:42:25.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:42:25.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:42:25.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.227.252:34209
23:42:25.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:42:25.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:42:25.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:42:25.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41091/jwks.json
23:42:25.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
23:42:25.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
23:42:25.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
23:42:25.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:42:25.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:42:25.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-2/data/info.pb
23:42:25.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.227.252
23:42:25.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:42:25.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:42:25.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:42:25.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:42:25.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:42:25.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:42:25.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-2/logs
23:42:25.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:42:25.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:42:25.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
23:42:25.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
23:42:25.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:42:25.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:42:25.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:42:25.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 20:03:34 UTC on ccycloud.quasar-agiwfo.root.comops.site
23:42:25.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4576543
23:42:25.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:42:25.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:25.771677 17498 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:42:25.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:25.773126 17498 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:42:25.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:25.804178 17490 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.227.252" port: 34209 } attempt: 5
23:42:25.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:25.816252 17490 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.227.252:34209: Network error: Client connection negotiation failed: client connection to 127.16.227.252:34209: connect: Connection refused (error 111)
23:42:25.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:25.830469 17512 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:42:26.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:26.312942 17398 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.227.252" port: 34209 } attempt: 6
23:42:26.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:26.335599 17398 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.227.252:34209: Network error: Client connection negotiation failed: client connection to 127.16.227.252:34209: connect: Connection refused (error 111)
23:42:26.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:26.364111 17490 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.227.252" port: 34209 } attempt: 6
23:42:26.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:26.385576 17490 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.227.252:34209: Network error: Client connection negotiation failed: client connection to 127.16.227.252:34209: connect: Connection refused (error 111)
23:42:27.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:25.839535 17515 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:42:27.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:25.832546 17513 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:42:27.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:27.235687 17514 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
23:42:27.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:27.235874 17498 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
23:42:27.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:27.239382 17498 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:42:27.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:27.242727 17498 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:42:27.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:27.244181 17498 hybrid_clock.cc:648] HybridClock initialized: now 1786318947244068 us; error 65 us; skew 500 ppm
23:42:27.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:27.245095 17498 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:42:27.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 23:42:27 dist-test-slave-8w9v krb5kdc[17308](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786318947, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.227.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
23:42:27.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:27.256938 17498 init.cc:377] Logged in from keytab as kudu/127.16.227.252@KRBTEST.COM (short username kudu)
23:42:27.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:27.269765 17498 webserver.cc:460] Webserver started at http://127.16.227.252:34955/ using document root <none> and password file <none>
23:42:27.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:27.270694 17498 fs_manager.cc:359] Metadata directory not provided
23:42:27.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:27.270897 17498 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:42:27.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:27.271302 17498 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
23:42:27.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:27.277285 17498 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-2/data/instance:
23:42:27.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1bdaf4a71b89405793c1b7bcb04cd5df"
23:42:27.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 23:42:27 on dist-test-slave-8w9v"
23:42:27.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:27.278357 17498 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-2/wal/instance:
23:42:27.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1bdaf4a71b89405793c1b7bcb04cd5df"
23:42:27.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 23:42:27 on dist-test-slave-8w9v"
23:42:27.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:27.289714 17498 fs_manager.cc:691] Time spent creating directory manager: real 0.011s user 0.010s sys 0.002s
23:42:27.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:27.299798 17531 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
23:42:27.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:27.301273 17498 fs_manager.cc:725] Time spent opening block manager: real 0.008s user 0.002s sys 0.005s
23:42:27.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:27.301543 17498 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-2/data,/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-2/wal
23:42:27.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1bdaf4a71b89405793c1b7bcb04cd5df"
23:42:27.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 23:42:27 on dist-test-slave-8w9v"
23:42:27.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:27.301888 17498 fs_report.cc:383] FS layout report
23:42:27.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:42:27.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-2/wal
23:42:27.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-2/wal
23:42:27.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-2/data/data
23:42:27.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:42:27.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:42:27.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:42:27.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:42:27.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:42:27.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:42:27.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:42:27.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:42:27.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:42:27.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:42:27.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:42:27.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:27.369199 17398 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.227.252" port: 34209 } attempt: 7
23:42:27.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:27.371892 17498 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
23:42:27.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:27.377435 17398 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.227.252:34209: Network error: Client connection negotiation failed: client connection to 127.16.227.252:34209: connect: Connection refused (error 111)
23:42:27.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:27.380069 17498 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:42:27.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:27.380838 17498 kserver.cc:163] Server-wide thread pool size limit: 3276
23:42:27.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:27.446334 17490 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.227.252" port: 34209 } attempt: 7
23:42:27.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:27.460768 17490 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.227.252:34209: Network error: Client connection negotiation failed: client connection to 127.16.227.252:34209: connect: Connection refused (error 111)
23:42:27.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:27.590756 17498 rpc_server.cc:307] RPC server started. Bound to: 127.16.227.252:34209
23:42:27.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:27.590860 17591 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.227.252:34209 every 8 connection(s)
23:42:27.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:27.594154 17498 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-2/data/info.pb
23:42:27.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:27.594949 17295 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskkf9L9h/build/tsan/bin/kudu as pid 17498
23:42:27.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:27.595269 17295 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-2/wal/instance
23:42:27.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:27.613794 17592 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
23:42:27.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:27.656858 17592 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.227.254" port: 42395 } has no permanent_uuid. Determining permanent_uuid...
23:42:27.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 23:42:27 dist-test-slave-8w9v krb5kdc[17308](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786318940, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.16.227.252@KRBTEST.COM
23:42:27.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 23:42:27 dist-test-slave-8w9v krb5kdc[17308](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786318947, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.227.252@KRBTEST.COM for kudu/127.16.227.254@KRBTEST.COM
23:42:27.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:27.709154 17592 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.227.253" port: 34331 } has no permanent_uuid. Determining permanent_uuid...
23:42:27.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 23:42:27 dist-test-slave-8w9v krb5kdc[17308](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786318947, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.227.252@KRBTEST.COM for kudu/127.16.227.253@KRBTEST.COM
23:42:27.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 23:42:27 dist-test-slave-8w9v krb5kdc[17308](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786318940, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.16.227.253@KRBTEST.COM
23:42:27.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:27.754645 17592 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.227.252" port: 34209 } has no permanent_uuid. Determining permanent_uuid...
23:42:27.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 23:42:27 dist-test-slave-8w9v krb5kdc[17308](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786318947, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.227.252@KRBTEST.COM for kudu/127.16.227.252@KRBTEST.COM
23:42:27.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 23:42:27 dist-test-slave-8w9v krb5kdc[17308](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786318940, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.16.227.254@KRBTEST.COM
23:42:27.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:27.797389 17592 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df: Bootstrap starting.
23:42:27.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:27.803010 17295 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
23:42:27.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:27.808236 17592 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df: Neither blocks nor log segments found. Creating new log.
23:42:27.810 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
23:42:27.812 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
23:42:27.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:27.811887 17592 log.cc:824] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df: Log is configured to *not* fsync() on all Append() calls
23:42:27.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:27.823423 17592 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df: No bootstrap required, opened a new log
23:42:27.829 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
23:42:27.831 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
23:42:27.834 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.16.227.254"
port: 42395
}
bound_http_address {
host: "127.16.227.254"
port: 41533
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.16.227.253"
port: 34331
}
bound_http_address {
host: "127.16.227.253"
port: 36583
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.16.227.252"
port: 34209
}
bound_http_address {
host: "127.16.227.252"
port: 34955
}
}
}
23:42:27.835 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
23:42:27.835 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
23:42:27.835 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
23:42:27.837 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
23:42:27.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:27.872496 17592 raft_consensus.cc:348] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa1cf38e72e643c39ce903ca13ecdd81" member_type: VOTER last_known_addr { host: "127.16.227.254" port: 42395 } } peers { permanent_uuid: "8fdb81d4101b4b3bb72f6e2f36f173c5" member_type: VOTER last_known_addr { host: "127.16.227.253" port: 34331 } } peers { permanent_uuid: "1bdaf4a71b89405793c1b7bcb04cd5df" member_type: VOTER last_known_addr { host: "127.16.227.252" port: 34209 } }
23:42:27.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:27.873407 17592 raft_consensus.cc:374] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
23:42:27.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:27.873768 17592 raft_consensus.cc:729] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1bdaf4a71b89405793c1b7bcb04cd5df, State: Initialized, Role: FOLLOWER
23:42:27.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:27.874874 17592 consensus_queue.cc:260] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa1cf38e72e643c39ce903ca13ecdd81" member_type: VOTER last_known_addr { host: "127.16.227.254" port: 42395 } } peers { permanent_uuid: "8fdb81d4101b4b3bb72f6e2f36f173c5" member_type: VOTER last_known_addr { host: "127.16.227.253" port: 34331 } } peers { permanent_uuid: "1bdaf4a71b89405793c1b7bcb04cd5df" member_type: VOTER last_known_addr { host: "127.16.227.252" port: 34209 } }
23:42:27.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:27.880470 17605 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa1cf38e72e643c39ce903ca13ecdd81" member_type: VOTER last_known_addr { host: "127.16.227.254" port: 42395 } } peers { permanent_uuid: "8fdb81d4101b4b3bb72f6e2f36f173c5" member_type: VOTER last_known_addr { host: "127.16.227.253" port: 34331 } } peers { permanent_uuid: "1bdaf4a71b89405793c1b7bcb04cd5df" member_type: VOTER last_known_addr { host: "127.16.227.252" port: 34209 } } }
23:42:27.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:27.882014 17605 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df [sys.catalog]: This master's current role is: FOLLOWER
23:42:27.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:27.883510 17592 sys_catalog.cc:564] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df [sys.catalog]: configured and running, proceeding with master startup.
23:42:27.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 23:42:27 dist-test-slave-8w9v krb5kdc[17308](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786318947, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
23:42:27.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
23:42:27.895 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
23:42:27.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:27.895804 17539 tablet.cc:2369] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df: Can't schedule compaction. Clean time has not been advanced past its initial value.
23:42:27.897 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/krb5kdc/krb5cc
23:42:27.909 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
23:42:27.911 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/krb5kdc/krb5cc
23:42:27.915 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
23:42:27.921 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
23:42:27.924 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
23:42:27.924 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
23:42:27.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 23:42:27 dist-test-slave-8w9v krb5kdc[17308](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786318947, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
23:42:27.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
23:42:27.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:27.975153 17617 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:42:27.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:27.975518 17617 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:42:27.984 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
23:42:27.984 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/krb5kdc/krb5cc
23:42:27.986 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
23:42:27.986 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.16.227.253:34331
23:42:27.988 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
23:42:27.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:27.989357 17295 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskkf9L9h/build/tsan/bin/kudu with pid 17405
23:42:28.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:28.004114 17605 raft_consensus.cc:482] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
23:42:28.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:28.004691 17605 raft_consensus.cc:504] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa1cf38e72e643c39ce903ca13ecdd81" member_type: VOTER last_known_addr { host: "127.16.227.254" port: 42395 } } peers { permanent_uuid: "8fdb81d4101b4b3bb72f6e2f36f173c5" member_type: VOTER last_known_addr { host: "127.16.227.253" port: 34331 } } peers { permanent_uuid: "1bdaf4a71b89405793c1b7bcb04cd5df" member_type: VOTER last_known_addr { host: "127.16.227.252" port: 34209 } }
23:42:28.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:27.958074 17498 master_runner.cc:186] Error getting master registration for 127.16.227.254:42395: OK, instance_id { permanent_uuid: "aa1cf38e72e643c39ce903ca13ecdd81" instance_seqno: 1786318942440604 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
23:42:28.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:28.008848 17498 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
23:42:28.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:28.009143 17605 leader_election.cc:288] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers aa1cf38e72e643c39ce903ca13ecdd81 (127.16.227.254:42395), 8fdb81d4101b4b3bb72f6e2f36f173c5 (127.16.227.253:34331)
23:42:28.024 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
23:42:28.025 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.16.227.254:42395
23:42:28.025 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
23:42:28.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:28.023229 17535 connection.cc:535] client connection to 127.16.227.253:34331 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
23:42:28.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:28.023720 17535 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
23:42:28.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:28.025840 17295 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskkf9L9h/build/tsan/bin/kudu with pid 17322
23:42:28.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:28.030380 17535 leader_election.cc:334] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 8fdb81d4101b4b3bb72f6e2f36f173c5 (127.16.227.253:34331): Network error: Client connection negotiation failed: client connection to 127.16.227.253:34331: connect: Connection refused (error 111)
23:42:28.056 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
23:42:28.056 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.16.227.252:34209
23:42:28.056 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
23:42:28.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:28.058084 17295 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskkf9L9h/build/tsan/bin/kudu with pid 17498
23:42:28.083 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
23:42:28.083 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.16.227.253:34331
23:42:28.086 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
23:42:28.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:28.088061 17295 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskkf9L9h/build/tsan/bin/kudu
23:42:28.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskkf9L9h/build/tsan/bin/kudu
23:42:28.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-1/wal
23:42:28.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-1/data
23:42:28.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:42:28.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:42:28.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:42:28.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:42:28.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:42:28.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:42:28.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-1/logs
23:42:28.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-1/data/info.pb
23:42:28.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:42:28.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:42:28.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:42:28.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:42:28.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:42:28.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:42:28.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:42:28.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:42:28.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
23:42:28.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:42:28.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:42:28.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:42:28.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.227.253:34331
23:42:28.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.227.253
23:42:28.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36583
23:42:28.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/krb5kdc/kudu_127.16.227.253.keytab
23:42:28.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.16.227.253
23:42:28.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
23:42:28.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
23:42:28.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
23:42:28.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.227.212:35697
23:42:28.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:42:28.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:42:28.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:42:28.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:42:28.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.227.254:42395,127.16.227.253:34331,127.16.227.252:34209
23:42:28.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:42:28.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41091/jwks.json
23:42:28.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
23:42:28.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:28.414281 17621 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:42:28.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:28.414810 17621 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:42:28.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:28.415408 17621 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:42:28.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:28.447191 17621 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
23:42:28.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:28.447420 17621 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
23:42:28.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:28.447679 17621 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:42:28.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:28.447870 17621 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
23:42:28.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:28.448056 17621 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
23:42:28.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:28.448329 17621 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
23:42:28.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:28.448551 17621 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41091/jwks.json
23:42:28.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:28.484807 17621 master_runner.cc:386] Master server non-default flags:
23:42:28.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:42:28.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.227.212:35697
23:42:28.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:42:28.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:42:28.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-1/data
23:42:28.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-1/wal
23:42:28.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
23:42:28.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:42:28.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.227.254:42395,127.16.227.253:34331,127.16.227.252:34209
23:42:28.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/krb5kdc/kudu_127.16.227.253.keytab
23:42:28.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.16.227.253
23:42:28.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:42:28.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:42:28.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:42:28.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.227.253:34331
23:42:28.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:42:28.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:42:28.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:42:28.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41091/jwks.json
23:42:28.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
23:42:28.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
23:42:28.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
23:42:28.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:42:28.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:42:28.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-1/data/info.pb
23:42:28.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.227.253
23:42:28.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36583
23:42:28.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:42:28.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:42:28.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:42:28.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:42:28.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:42:28.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-1/logs
23:42:28.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:42:28.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:42:28.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
23:42:28.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
23:42:28.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:42:28.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:42:28.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:42:28.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 20:03:34 UTC on ccycloud.quasar-agiwfo.root.comops.site
23:42:28.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4576543
23:42:28.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:42:28.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:28.486078 17621 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:42:28.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:28.487111 17621 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:42:28.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:28.504838 17629 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:42:29.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:28.504907 17631 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:42:29.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:28.507462 17621 server_base.cc:1034] running on GCE node
23:42:29.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:28.506585 17628 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:42:29.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:29.698709 17621 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:42:29.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:29.701620 17621 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:42:29.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:29.703017 17621 hybrid_clock.cc:648] HybridClock initialized: now 1786318949702972 us; error 59 us; skew 500 ppm
23:42:29.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:29.704002 17621 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:42:29.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 23:42:29 dist-test-slave-8w9v krb5kdc[17308](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786318949, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.227.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
23:42:29.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:29.710994 17621 init.cc:377] Logged in from keytab as kudu/127.16.227.253@KRBTEST.COM (short username kudu)
23:42:29.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:29.719457 17621 webserver.cc:460] Webserver started at http://127.16.227.253:36583/ using document root <none> and password file <none>
23:42:29.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:29.720633 17621 fs_manager.cc:359] Metadata directory not provided
23:42:29.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:29.720899 17621 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:42:29.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:29.731212 17621 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.004s sys 0.001s
23:42:29.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:29.735762 17640 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
23:42:29.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:29.736864 17621 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
23:42:29.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:29.737142 17621 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-1/data,/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-1/wal
23:42:29.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8fdb81d4101b4b3bb72f6e2f36f173c5"
23:42:29.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 23:42:24 on dist-test-slave-8w9v"
23:42:29.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:29.739387 17621 fs_report.cc:383] FS layout report
23:42:29.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:42:29.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-1/wal
23:42:29.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-1/wal
23:42:29.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-1/data/data
23:42:29.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:42:29.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:42:29.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:42:29.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:42:29.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:42:29.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:42:29.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:42:29.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:42:29.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:42:29.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:42:29.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:42:29.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:29.781585 17621 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
23:42:29.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:29.787842 17621 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:42:29.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:29.788369 17621 kserver.cc:163] Server-wide thread pool size limit: 3276
23:42:29.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:29.857863 17621 rpc_server.cc:307] RPC server started. Bound to: 127.16.227.253:34331
23:42:29.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:29.857997 17700 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.227.253:34331 every 8 connection(s)
23:42:29.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:29.860754 17621 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-1/data/info.pb
23:42:29.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:29.865236 17295 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskkf9L9h/build/tsan/bin/kudu as pid 17621
23:42:29.866 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
23:42:29.866 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.16.227.254:42395
23:42:29.866 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
23:42:29.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:29.867659 17295 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskkf9L9h/build/tsan/bin/kudu
23:42:29.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskkf9L9h/build/tsan/bin/kudu
23:42:29.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-0/wal
23:42:29.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-0/data
23:42:29.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:42:29.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:42:29.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:42:29.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:42:29.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:42:29.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:42:29.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-0/logs
23:42:29.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-0/data/info.pb
23:42:29.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:42:29.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:42:29.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:42:29.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:42:29.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:42:29.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:42:29.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:42:29.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:42:29.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
23:42:29.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:42:29.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:42:29.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:42:29.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.227.254:42395
23:42:29.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.227.254
23:42:29.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41533
23:42:29.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/krb5kdc/kudu_127.16.227.254.keytab
23:42:29.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.16.227.254
23:42:29.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
23:42:29.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
23:42:29.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
23:42:29.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.227.212:35697
23:42:29.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:42:29.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:42:29.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:42:29.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:42:29.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.227.254:42395,127.16.227.253:34331,127.16.227.252:34209
23:42:29.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:42:29.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41091/jwks.json
23:42:29.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
23:42:29.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:29.874136 17701 sys_catalog.cc:263] Verifying existing consensus state
23:42:29.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260809 23:42:29.875181 17701 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
23:42:29.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:29.875576 17621 master.cc:537] Master@127.16.227.253:34331 shutting down...
23:42:29.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:29.938359 17621 master.cc:552] Master@127.16.227.253:34331 shutdown complete.
23:42:29.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
23:42:30.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:30.203250 17703 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:42:30.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:30.204006 17703 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:42:30.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:30.204767 17703 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:42:30.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:30.234439 17703 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
23:42:30.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:30.234647 17703 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
23:42:30.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:30.234897 17703 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:42:30.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:30.235178 17703 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
23:42:30.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:30.235380 17703 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
23:42:30.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:30.235563 17703 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
23:42:30.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:30.235790 17703 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41091/jwks.json
23:42:30.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:30.275002 17703 master_runner.cc:386] Master server non-default flags:
23:42:30.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:42:30.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.227.212:35697
23:42:30.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:42:30.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:42:30.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-0/data
23:42:30.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-0/wal
23:42:30.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
23:42:30.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:42:30.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.227.254:42395,127.16.227.253:34331,127.16.227.252:34209
23:42:30.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/krb5kdc/kudu_127.16.227.254.keytab
23:42:30.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.16.227.254
23:42:30.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:42:30.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:42:30.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:42:30.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.227.254:42395
23:42:30.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:42:30.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:42:30.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:42:30.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41091/jwks.json
23:42:30.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
23:42:30.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
23:42:30.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
23:42:30.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:42:30.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:42:30.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-0/data/info.pb
23:42:30.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.227.254
23:42:30.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41533
23:42:30.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:42:30.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:42:30.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:42:30.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:42:30.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:42:30.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-0/logs
23:42:30.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:42:30.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:42:30.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
23:42:30.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
23:42:30.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:42:30.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:42:30.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:42:30.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 20:03:34 UTC on ccycloud.quasar-agiwfo.root.comops.site
23:42:30.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4576543
23:42:30.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:42:30.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:30.276216 17703 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:42:30.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:30.277235 17703 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:42:30.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:30.295058 17711 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:42:31.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:30.295225 17712 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:42:31.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:30.296841 17703 server_base.cc:1034] running on GCE node
23:42:31.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:30.296597 17714 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:42:31.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:31.489099 17703 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:42:31.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:31.492408 17703 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:42:31.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:31.493826 17703 hybrid_clock.cc:648] HybridClock initialized: now 1786318951493788 us; error 58 us; skew 500 ppm
23:42:31.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:31.494589 17703 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:42:31.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 23:42:31 dist-test-slave-8w9v krb5kdc[17308](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786318951, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.227.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
23:42:31.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:31.500634 17703 init.cc:377] Logged in from keytab as kudu/127.16.227.254@KRBTEST.COM (short username kudu)
23:42:31.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:31.505874 17703 webserver.cc:460] Webserver started at http://127.16.227.254:41533/ using document root <none> and password file <none>
23:42:31.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:31.506721 17703 fs_manager.cc:359] Metadata directory not provided
23:42:31.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:31.506906 17703 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:42:31.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:31.514855 17703 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.005s sys 0.000s
23:42:31.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:31.519078 17724 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
23:42:31.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:31.520012 17703 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
23:42:31.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:31.520431 17703 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-0/data,/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-0/wal
23:42:31.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aa1cf38e72e643c39ce903ca13ecdd81"
23:42:31.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 23:42:22 on dist-test-slave-8w9v"
23:42:31.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:31.522722 17703 fs_report.cc:383] FS layout report
23:42:31.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:42:31.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-0/wal
23:42:31.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-0/wal
23:42:31.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-0/data/data
23:42:31.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:42:31.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:42:31.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:42:31.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:42:31.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:42:31.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:42:31.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:42:31.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:42:31.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:42:31.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:42:31.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:42:31.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:31.572422 17703 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
23:42:31.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:31.579306 17703 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:42:31.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:31.579734 17703 kserver.cc:163] Server-wide thread pool size limit: 3276
23:42:31.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:31.652271 17703 rpc_server.cc:307] RPC server started. Bound to: 127.16.227.254:42395
23:42:31.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:31.652366 17782 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.227.254:42395 every 8 connection(s)
23:42:31.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:31.655009 17703 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-0/data/info.pb
23:42:31.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:31.663245 17295 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskkf9L9h/build/tsan/bin/kudu as pid 17703
23:42:31.664 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
23:42:31.664 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.16.227.252:34209
23:42:31.664 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
23:42:31.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:31.665210 17783 sys_catalog.cc:263] Verifying existing consensus state
23:42:31.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260809 23:42:31.666002 17783 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
23:42:31.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:31.666055 17295 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskkf9L9h/build/tsan/bin/kudu
23:42:31.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskkf9L9h/build/tsan/bin/kudu
23:42:31.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-2/wal
23:42:31.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-2/data
23:42:31.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:42:31.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:42:31.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:42:31.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:42:31.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:42:31.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:42:31.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-2/logs
23:42:31.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-2/data/info.pb
23:42:31.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:42:31.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:42:31.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:42:31.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:42:31.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:42:31.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:42:31.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:42:31.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:42:31.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
23:42:31.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:42:31.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:42:31.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:42:31.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.227.252:34209
23:42:31.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.227.252
23:42:31.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34955
23:42:31.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/krb5kdc/kudu_127.16.227.252.keytab
23:42:31.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.16.227.252
23:42:31.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
23:42:31.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
23:42:31.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
23:42:31.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.227.212:35697
23:42:31.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:42:31.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:42:31.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:42:31.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:42:31.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.227.254:42395,127.16.227.253:34331,127.16.227.252:34209
23:42:31.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:42:31.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41091/jwks.json
23:42:31.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
23:42:31.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:31.666508 17703 master.cc:537] Master@127.16.227.254:42395 shutting down...
23:42:31.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:31.730298 17703 master.cc:552] Master@127.16.227.254:42395 shutdown complete.
23:42:31.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
23:42:31.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:31.992770 17785 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:42:31.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:31.993312 17785 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:42:31.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:31.993883 17785 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:42:32.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:32.028107 17785 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
23:42:32.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:32.028364 17785 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
23:42:32.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:32.028673 17785 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:42:32.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:32.028896 17785 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
23:42:32.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:32.029071 17785 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
23:42:32.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:32.029280 17785 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
23:42:32.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:32.029452 17785 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41091/jwks.json
23:42:32.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:32.066303 17785 master_runner.cc:386] Master server non-default flags:
23:42:32.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:42:32.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.227.212:35697
23:42:32.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:42:32.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:42:32.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-2/data
23:42:32.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-2/wal
23:42:32.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
23:42:32.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:42:32.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.227.254:42395,127.16.227.253:34331,127.16.227.252:34209
23:42:32.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/krb5kdc/kudu_127.16.227.252.keytab
23:42:32.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.16.227.252
23:42:32.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:42:32.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:42:32.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:42:32.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.227.252:34209
23:42:32.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:42:32.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:42:32.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:42:32.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41091/jwks.json
23:42:32.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
23:42:32.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
23:42:32.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
23:42:32.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:42:32.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:42:32.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-2/data/info.pb
23:42:32.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.227.252
23:42:32.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34955
23:42:32.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:42:32.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:42:32.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:42:32.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:42:32.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:42:32.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-2/logs
23:42:32.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:42:32.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:42:32.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
23:42:32.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
23:42:32.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:42:32.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:42:32.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:42:32.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 20:03:34 UTC on ccycloud.quasar-agiwfo.root.comops.site
23:42:32.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4576543
23:42:32.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:42:32.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:32.067581 17785 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:42:32.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:32.068710 17785 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:42:32.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:32.086076 17793 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:42:33.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:32.087445 17795 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:42:33.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:32.087064 17792 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:42:33.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:33.400379 17794 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1311 milliseconds
23:42:33.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:33.400508 17785 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
23:42:33.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:33.401821 17785 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:42:33.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:33.404604 17785 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:42:33.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:33.406044 17785 hybrid_clock.cc:648] HybridClock initialized: now 1786318953406004 us; error 35 us; skew 500 ppm
23:42:33.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:33.406910 17785 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:42:33.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 23:42:33 dist-test-slave-8w9v krb5kdc[17308](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786318953, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.227.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
23:42:33.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:33.417429 17785 init.cc:377] Logged in from keytab as kudu/127.16.227.252@KRBTEST.COM (short username kudu)
23:42:33.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:33.422315 17785 webserver.cc:460] Webserver started at http://127.16.227.252:34955/ using document root <none> and password file <none>
23:42:33.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:33.423260 17785 fs_manager.cc:359] Metadata directory not provided
23:42:33.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:33.423513 17785 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:42:33.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:33.432789 17785 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.007s sys 0.001s
23:42:33.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:33.437916 17805 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
23:42:33.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:33.438913 17785 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
23:42:33.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:33.439249 17785 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-2/data,/tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-2/wal
23:42:33.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1bdaf4a71b89405793c1b7bcb04cd5df"
23:42:33.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 23:42:27 on dist-test-slave-8w9v"
23:42:33.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:33.441593 17785 fs_report.cc:383] FS layout report
23:42:33.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:42:33.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-2/wal
23:42:33.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-2/wal
23:42:33.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-2/data/data
23:42:33.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:42:33.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:42:33.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:42:33.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:42:33.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:42:33.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:42:33.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:42:33.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:42:33.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:42:33.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:42:33.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:42:33.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:33.483373 17785 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
23:42:33.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:33.489506 17785 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:42:33.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:33.490005 17785 kserver.cc:163] Server-wide thread pool size limit: 3276
23:42:33.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:33.565510 17785 rpc_server.cc:307] RPC server started. Bound to: 127.16.227.252:34209
23:42:33.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:33.565653 17861 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.227.252:34209 every 8 connection(s)
23:42:33.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:33.568418 17785 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskkf9L9h/test-tmp/mini-kudu-cluster9211570069294879065/master-2/data/info.pb
23:42:33.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:33.578739 17295 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskkf9L9h/build/tsan/bin/kudu as pid 17785
23:42:33.579 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
23:42:33.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:33.580224 17862 sys_catalog.cc:263] Verifying existing consensus state
23:42:33.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:33.588469 17862 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df: Bootstrap starting.
23:42:33.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:33.633687 17862 log.cc:824] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df: Log is configured to *not* fsync() on all Append() calls
23:42:33.640 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.16.227.253:34331(127.16.227.253:34331)] connecting to peer
23:42:33.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:33.640007 17862 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
23:42:33.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:33.641073 17862 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df: Bootstrap complete.
23:42:33.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:33.675544 17862 raft_consensus.cc:348] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa1cf38e72e643c39ce903ca13ecdd81" member_type: VOTER last_known_addr { host: "127.16.227.254" port: 42395 } } peers { permanent_uuid: "8fdb81d4101b4b3bb72f6e2f36f173c5" member_type: VOTER last_known_addr { host: "127.16.227.253" port: 34331 } } peers { permanent_uuid: "1bdaf4a71b89405793c1b7bcb04cd5df" member_type: VOTER last_known_addr { host: "127.16.227.252" port: 34209 } }
23:42:33.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:33.679060 17862 raft_consensus.cc:374] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
23:42:33.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:33.679401 17862 raft_consensus.cc:729] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1bdaf4a71b89405793c1b7bcb04cd5df, State: Initialized, Role: FOLLOWER
23:42:33.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:33.680431 17862 consensus_queue.cc:260] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa1cf38e72e643c39ce903ca13ecdd81" member_type: VOTER last_known_addr { host: "127.16.227.254" port: 42395 } } peers { permanent_uuid: "8fdb81d4101b4b3bb72f6e2f36f173c5" member_type: VOTER last_known_addr { host: "127.16.227.253" port: 34331 } } peers { permanent_uuid: "1bdaf4a71b89405793c1b7bcb04cd5df" member_type: VOTER last_known_addr { host: "127.16.227.252" port: 34209 } }
23:42:33.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:33.683442 17866 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa1cf38e72e643c39ce903ca13ecdd81" member_type: VOTER last_known_addr { host: "127.16.227.254" port: 42395 } } peers { permanent_uuid: "8fdb81d4101b4b3bb72f6e2f36f173c5" member_type: VOTER last_known_addr { host: "127.16.227.253" port: 34331 } } peers { permanent_uuid: "1bdaf4a71b89405793c1b7bcb04cd5df" member_type: VOTER last_known_addr { host: "127.16.227.252" port: 34209 } } }
23:42:33.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:33.685014 17866 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df [sys.catalog]: This master's current role is: FOLLOWER
23:42:33.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:33.685223 17862 sys_catalog.cc:564] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df [sys.catalog]: configured and running, proceeding with master startup.
23:42:33.738 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.16.227.254:42395(127.16.227.254:42395)] connecting to peer
23:42:33.740 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.16.227.252:34209(127.16.227.252:34209)] connecting to peer
23:42:33.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:33.745457 17813 tablet.cc:2369] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df: Can't schedule compaction. Clean time has not been advanced past its initial value.
23:42:33.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:33.757669 17879 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:42:33.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:33.757992 17879 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:42:33.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:33.755285 17785 master_runner.cc:186] Error getting master registration for 127.16.227.254:42395: Network error: Client connection negotiation failed: client connection to 127.16.227.254:42395: connect: Connection refused (error 111),
23:42:33.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:33.783143 17785 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
23:42:33.804 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:33.804 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.227.254:42395(127.16.227.254:42395)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:33.806 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.227.254:42395: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:33.807 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:33.808 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.227.253:34331(127.16.227.253:34331)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:33.809 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.227.253:34331: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:33.809 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.16.227.252:34209(127.16.227.252:34209)] Successfully connected to peer
23:42:33.902 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
23:42:33.913 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.16.227.252
23:42:34.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:34.105397 17866 raft_consensus.cc:482] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
23:42:34.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:34.105965 17866 raft_consensus.cc:504] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa1cf38e72e643c39ce903ca13ecdd81" member_type: VOTER last_known_addr { host: "127.16.227.254" port: 42395 } } peers { permanent_uuid: "8fdb81d4101b4b3bb72f6e2f36f173c5" member_type: VOTER last_known_addr { host: "127.16.227.253" port: 34331 } } peers { permanent_uuid: "1bdaf4a71b89405793c1b7bcb04cd5df" member_type: VOTER last_known_addr { host: "127.16.227.252" port: 34209 } }
23:42:34.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:34.119643 17807 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.227.254:42395: connect: Connection refused (error 111)
23:42:34.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:34.119338 17866 leader_election.cc:288] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers aa1cf38e72e643c39ce903ca13ecdd81 (127.16.227.254:42395), 8fdb81d4101b4b3bb72f6e2f36f173c5 (127.16.227.253:34331)
23:42:34.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 23:42:34 dist-test-slave-8w9v krb5kdc[17308](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786318947, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.16.227.252@KRBTEST.COM
23:42:34.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:34.150477 17807 leader_election.cc:334] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer aa1cf38e72e643c39ce903ca13ecdd81 (127.16.227.254:42395): Network error: Client connection negotiation failed: client connection to 127.16.227.254:42395: connect: Connection refused (error 111)
23:42:34.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:34.152460 17810 leader_election.cc:334] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 8fdb81d4101b4b3bb72f6e2f36f173c5 (127.16.227.253:34331): Network error: Client connection negotiation failed: client connection to 127.16.227.253:34331: connect: Connection refused (error 111)
23:42:34.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:34.152899 17810 leader_election.cc:302] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1bdaf4a71b89405793c1b7bcb04cd5df; no voters: 8fdb81d4101b4b3bb72f6e2f36f173c5, aa1cf38e72e643c39ce903ca13ecdd81
23:42:34.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:34.154031 17866 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
23:42:34.175 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x997742da, L:/127.0.0.1:39678 - R:/127.16.227.252:34209] using SASL/GSSAPI
23:42:34.195 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209; will retry
23:42:34.206 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.227.253:34331(127.16.227.253:34331)] connecting to peer
23:42:34.209 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:34.209 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.227.253:34331(127.16.227.253:34331)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:34.211 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.227.253:34331: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:34.211 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.227.254:42395(127.16.227.254:42395)] connecting to peer
23:42:34.216 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:34.216 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.227.254:42395(127.16.227.254:42395)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:34.217 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.227.254:42395: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:34.219 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209; will retry
23:42:34.225 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.227.253:34331(127.16.227.253:34331)] connecting to peer
23:42:34.226 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.227.254:42395(127.16.227.254:42395)] connecting to peer
23:42:34.229 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:34.229 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.227.253:34331(127.16.227.253:34331)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:34.230 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.227.253:34331: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:34.231 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:34.231 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.227.254:42395(127.16.227.254:42395)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:34.232 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.227.254:42395: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:34.234 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209; will retry
23:42:34.245 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.227.253:34331(127.16.227.253:34331)] connecting to peer
23:42:34.247 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.227.254:42395(127.16.227.254:42395)] connecting to peer
23:42:34.247 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:34.247 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.227.253:34331(127.16.227.253:34331)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:34.248 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.227.253:34331: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:34.250 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:34.250 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.227.254:42395(127.16.227.254:42395)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:34.252 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.227.254:42395: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:34.254 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209; will retry
23:42:34.285 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.227.253:34331(127.16.227.253:34331)] connecting to peer
23:42:34.286 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.227.254:42395(127.16.227.254:42395)] connecting to peer
23:42:34.288 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:34.288 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.227.253:34331(127.16.227.253:34331)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:34.289 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.227.253:34331: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:34.290 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:34.291 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.227.254:42395(127.16.227.254:42395)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:34.291 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.227.254:42395: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:34.293 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209; will retry
23:42:34.325 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.227.253:34331(127.16.227.253:34331)] connecting to peer
23:42:34.327 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.227.254:42395(127.16.227.254:42395)] connecting to peer
23:42:34.328 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:34.329 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.227.253:34331(127.16.227.253:34331)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:34.329 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.227.253:34331: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:34.332 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:34.333 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.227.254:42395(127.16.227.254:42395)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:34.333 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.227.254:42395: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:34.335 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209; will retry
23:42:34.386 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.227.253:34331(127.16.227.253:34331)] connecting to peer
23:42:34.387 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.227.254:42395(127.16.227.254:42395)] connecting to peer
23:42:34.388 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:34.389 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.227.253:34331(127.16.227.253:34331)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:34.389 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.227.253:34331: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:34.392 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:34.392 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.227.254:42395(127.16.227.254:42395)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:34.393 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.227.254:42395: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:34.394 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209; will retry
23:42:34.505 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.227.253:34331(127.16.227.253:34331)] connecting to peer
23:42:34.507 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.227.254:42395(127.16.227.254:42395)] connecting to peer
23:42:34.508 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:34.508 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.227.253:34331(127.16.227.253:34331)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:34.509 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.227.253:34331: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:34.510 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:34.510 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.227.254:42395(127.16.227.254:42395)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:34.510 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.227.254:42395: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:34.513 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209; will retry
23:42:34.545 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.227.253:34331(127.16.227.253:34331)] connecting to peer
23:42:34.546 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.227.254:42395(127.16.227.254:42395)] connecting to peer
23:42:34.556 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:34.556 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.227.253:34331(127.16.227.253:34331)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:34.561 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.227.253:34331: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:34.572 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:34.573 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.227.254:42395(127.16.227.254:42395)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:34.573 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.227.254:42395: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:34.574 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209; will retry
23:42:34.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:34.761207 17879 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:42:34.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:34.761471 17879 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:42:34.906 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.227.253:34331(127.16.227.253:34331)] connecting to peer
23:42:34.907 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.227.254:42395(127.16.227.254:42395)] connecting to peer
23:42:34.908 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:34.909 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.227.253:34331(127.16.227.253:34331)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:34.909 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.227.253:34331: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:34.910 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:34.911 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.227.254:42395(127.16.227.254:42395)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:34.912 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.227.254:42395: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:34.913 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209; will retry
23:42:35.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:35.664793 17892 raft_consensus.cc:482] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
23:42:35.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:35.665153 17892 raft_consensus.cc:504] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa1cf38e72e643c39ce903ca13ecdd81" member_type: VOTER last_known_addr { host: "127.16.227.254" port: 42395 } } peers { permanent_uuid: "8fdb81d4101b4b3bb72f6e2f36f173c5" member_type: VOTER last_known_addr { host: "127.16.227.253" port: 34331 } } peers { permanent_uuid: "1bdaf4a71b89405793c1b7bcb04cd5df" member_type: VOTER last_known_addr { host: "127.16.227.252" port: 34209 } }
23:42:35.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:35.666652 17892 leader_election.cc:288] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers aa1cf38e72e643c39ce903ca13ecdd81 (127.16.227.254:42395), 8fdb81d4101b4b3bb72f6e2f36f173c5 (127.16.227.253:34331)
23:42:35.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:35.672616 17810 leader_election.cc:334] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 8fdb81d4101b4b3bb72f6e2f36f173c5 (127.16.227.253:34331): Network error: Client connection negotiation failed: client connection to 127.16.227.253:34331: connect: Connection refused (error 111)
23:42:35.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:35.673004 17807 leader_election.cc:334] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer aa1cf38e72e643c39ce903ca13ecdd81 (127.16.227.254:42395): Network error: Client connection negotiation failed: client connection to 127.16.227.254:42395: connect: Connection refused (error 111)
23:42:35.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:35.673301 17807 leader_election.cc:302] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1bdaf4a71b89405793c1b7bcb04cd5df; no voters: 8fdb81d4101b4b3bb72f6e2f36f173c5, aa1cf38e72e643c39ce903ca13ecdd81
23:42:35.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:35.673810 17892 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
23:42:35.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:35.764791 17879 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:42:35.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:35.765028 17879 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:42:35.866 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.227.253:34331(127.16.227.253:34331)] connecting to peer
23:42:35.867 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.227.254:42395(127.16.227.254:42395)] connecting to peer
23:42:35.867 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:35.868 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.227.253:34331(127.16.227.253:34331)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:35.868 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.227.253:34331: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:35.869 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:35.869 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.227.254:42395(127.16.227.254:42395)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:35.870 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.227.254:42395: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:35.872 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209; will retry
23:42:36.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:36.767592 17879 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:42:36.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:36.767839 17879 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:42:37.345 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.227.253:34331(127.16.227.253:34331)] connecting to peer
23:42:37.346 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.227.254:42395(127.16.227.254:42395)] connecting to peer
23:42:37.347 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:37.347 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.227.253:34331(127.16.227.253:34331)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:37.348 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:37.348 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.227.253:34331: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:37.348 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.227.254:42395(127.16.227.254:42395)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:37.349 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.227.254:42395: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:37.353 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209; will retry
23:42:37.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:37.770594 17879 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:42:37.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:37.770854 17879 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:42:38.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:38.703501 17903 raft_consensus.cc:482] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
23:42:38.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:38.703922 17903 raft_consensus.cc:504] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa1cf38e72e643c39ce903ca13ecdd81" member_type: VOTER last_known_addr { host: "127.16.227.254" port: 42395 } } peers { permanent_uuid: "8fdb81d4101b4b3bb72f6e2f36f173c5" member_type: VOTER last_known_addr { host: "127.16.227.253" port: 34331 } } peers { permanent_uuid: "1bdaf4a71b89405793c1b7bcb04cd5df" member_type: VOTER last_known_addr { host: "127.16.227.252" port: 34209 } }
23:42:38.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:38.705610 17903 leader_election.cc:288] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers aa1cf38e72e643c39ce903ca13ecdd81 (127.16.227.254:42395), 8fdb81d4101b4b3bb72f6e2f36f173c5 (127.16.227.253:34331)
23:42:38.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:38.712806 17807 leader_election.cc:334] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer aa1cf38e72e643c39ce903ca13ecdd81 (127.16.227.254:42395): Network error: Client connection negotiation failed: client connection to 127.16.227.254:42395: connect: Connection refused (error 111)
23:42:38.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:38.713218 17810 leader_election.cc:334] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 8fdb81d4101b4b3bb72f6e2f36f173c5 (127.16.227.253:34331): Network error: Client connection negotiation failed: client connection to 127.16.227.253:34331: connect: Connection refused (error 111)
23:42:38.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:38.713616 17810 leader_election.cc:302] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1bdaf4a71b89405793c1b7bcb04cd5df; no voters: 8fdb81d4101b4b3bb72f6e2f36f173c5, aa1cf38e72e643c39ce903ca13ecdd81
23:42:38.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:38.714239 17903 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
23:42:38.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:38.773595 17879 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:42:38.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:38.773901 17879 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:42:38.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:38.785475 17785 master_runner.cc:186] Error getting master registration for 127.16.227.254:42395: Network error: Client connection negotiation failed: client connection to 127.16.227.254:42395: connect: Connection refused (error 111),
23:42:38.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:38.785928 17785 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
23:42:39.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:39.776764 17879 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:42:39.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:39.777053 17879 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:42:39.825 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.227.253:34331(127.16.227.253:34331)] connecting to peer
23:42:39.826 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.227.254:42395(127.16.227.254:42395)] connecting to peer
23:42:39.827 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:39.827 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.227.253:34331(127.16.227.253:34331)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:39.828 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.227.253:34331: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:39.829 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:39.829 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.227.254:42395(127.16.227.254:42395)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:39.829 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.227.254:42395: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:39.832 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209; will retry
23:42:40.145 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.227.253:34331(127.16.227.253:34331)] connecting to peer
23:42:40.146 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.227.254:42395(127.16.227.254:42395)] connecting to peer
23:42:40.147 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:40.147 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.227.253:34331(127.16.227.253:34331)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:40.147 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.227.253:34331: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:40.148 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:40.149 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.227.254:42395(127.16.227.254:42395)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:40.149 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.227.254:42395: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:40.151 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209; will retry
23:42:40.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:40.779922 17879 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:42:40.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:40.780248 17879 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:42:41.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:41.783061 17879 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:42:41.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:41.783404 17879 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:42:42.026 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.227.253:34331(127.16.227.253:34331)] connecting to peer
23:42:42.027 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.227.254:42395(127.16.227.254:42395)] connecting to peer
23:42:42.027 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:42.028 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.227.253:34331(127.16.227.253:34331)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:42.028 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.227.253:34331: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:42.029 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:42.029 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.227.254:42395(127.16.227.254:42395)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:42.030 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.227.254:42395: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:42.032 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209; will retry
23:42:42.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:42.056828 17910 raft_consensus.cc:482] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
23:42:42.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:42.057394 17910 raft_consensus.cc:504] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa1cf38e72e643c39ce903ca13ecdd81" member_type: VOTER last_known_addr { host: "127.16.227.254" port: 42395 } } peers { permanent_uuid: "8fdb81d4101b4b3bb72f6e2f36f173c5" member_type: VOTER last_known_addr { host: "127.16.227.253" port: 34331 } } peers { permanent_uuid: "1bdaf4a71b89405793c1b7bcb04cd5df" member_type: VOTER last_known_addr { host: "127.16.227.252" port: 34209 } }
23:42:42.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:42.060377 17910 leader_election.cc:288] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers aa1cf38e72e643c39ce903ca13ecdd81 (127.16.227.254:42395), 8fdb81d4101b4b3bb72f6e2f36f173c5 (127.16.227.253:34331)
23:42:42.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:42.065112 17807 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.227.254:42395: connect: Connection refused (error 111) [suppressed 5 similar messages]
23:42:42.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:42.078090 17807 leader_election.cc:334] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer aa1cf38e72e643c39ce903ca13ecdd81 (127.16.227.254:42395): Network error: Client connection negotiation failed: client connection to 127.16.227.254:42395: connect: Connection refused (error 111)
23:42:42.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:42.080467 17810 leader_election.cc:334] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 8fdb81d4101b4b3bb72f6e2f36f173c5 (127.16.227.253:34331): Network error: Client connection negotiation failed: client connection to 127.16.227.253:34331: connect: Connection refused (error 111)
23:42:42.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:42.081053 17810 leader_election.cc:302] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1bdaf4a71b89405793c1b7bcb04cd5df; no voters: 8fdb81d4101b4b3bb72f6e2f36f173c5, aa1cf38e72e643c39ce903ca13ecdd81
23:42:42.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:42.081900 17910 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
23:42:42.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:42.787848 17879 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:42:42.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:42.788233 17879 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:42:43.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:43.117923 17129 debug-util.cc:398] Leaking SignalData structure 0x7b08000af5c0 after lost signal to thread 16998
23:42:43.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:43.118791 17129 debug-util.cc:398] Leaking SignalData structure 0x7b08000af3a0 after lost signal to thread 17132
23:42:43.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 23:42:43 dist-test-slave-8w9v krb5kdc[17308](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786318963, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.227.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
23:42:43.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:43.421569 17801 init.cc:295] Successfully reacquired a new kerberos TGT
23:42:43.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:43.789966 17785 master_runner.cc:186] Error getting master registration for 127.16.227.254:42395: Network error: Client connection negotiation failed: client connection to 127.16.227.254:42395: connect: Connection refused (error 111),
23:42:43.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:43.790405 17785 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
23:42:43.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:43.792716 17879 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:42:43.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:43.793030 17879 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:42:44.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:44.513412 17916 raft_consensus.cc:482] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
23:42:44.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:44.513863 17916 raft_consensus.cc:504] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa1cf38e72e643c39ce903ca13ecdd81" member_type: VOTER last_known_addr { host: "127.16.227.254" port: 42395 } } peers { permanent_uuid: "8fdb81d4101b4b3bb72f6e2f36f173c5" member_type: VOTER last_known_addr { host: "127.16.227.253" port: 34331 } } peers { permanent_uuid: "1bdaf4a71b89405793c1b7bcb04cd5df" member_type: VOTER last_known_addr { host: "127.16.227.252" port: 34209 } }
23:42:44.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:44.515627 17916 leader_election.cc:288] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers aa1cf38e72e643c39ce903ca13ecdd81 (127.16.227.254:42395), 8fdb81d4101b4b3bb72f6e2f36f173c5 (127.16.227.253:34331)
23:42:44.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:44.520689 17807 leader_election.cc:334] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer aa1cf38e72e643c39ce903ca13ecdd81 (127.16.227.254:42395): Network error: Client connection negotiation failed: client connection to 127.16.227.254:42395: connect: Connection refused (error 111)
23:42:44.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:44.521402 17810 leader_election.cc:334] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 8fdb81d4101b4b3bb72f6e2f36f173c5 (127.16.227.253:34331): Network error: Client connection negotiation failed: client connection to 127.16.227.253:34331: connect: Connection refused (error 111)
23:42:44.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:44.521768 17810 leader_election.cc:302] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1bdaf4a71b89405793c1b7bcb04cd5df; no voters: 8fdb81d4101b4b3bb72f6e2f36f173c5, aa1cf38e72e643c39ce903ca13ecdd81
23:42:44.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:44.522908 17916 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
23:42:44.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:44.795984 17879 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:42:44.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:44.796271 17879 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:42:44.985 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.227.253:34331(127.16.227.253:34331)] connecting to peer
23:42:44.986 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.227.254:42395(127.16.227.254:42395)] connecting to peer
23:42:44.989 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:44.990 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.227.254:42395(127.16.227.254:42395)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:44.990 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.227.254:42395: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:44.990 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:44.990 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.227.253:34331(127.16.227.253:34331)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:44.991 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.227.253:34331: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:44.992 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209; will retry
23:42:45.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:45.798982 17879 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:42:45.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:45.799224 17879 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:42:46.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:46.802206 17879 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:42:46.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:46.802558 17879 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:42:47.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:47.805626 17879 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:42:47.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:47.805927 17879 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:42:48.526 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.227.253:34331(127.16.227.253:34331)] connecting to peer
23:42:48.527 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.227.254:42395(127.16.227.254:42395)] connecting to peer
23:42:48.532 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:48.532 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.227.253:34331(127.16.227.253:34331)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:48.532 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.227.253:34331: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:48.533 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:48.533 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.227.254:42395(127.16.227.254:42395)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:48.533 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.227.254:42395: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:48.535 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209; will retry
23:42:48.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:48.793084 17785 master_runner.cc:186] Error getting master registration for 127.16.227.254:42395: Network error: Client connection negotiation failed: client connection to 127.16.227.254:42395: connect: Connection refused (error 111),
23:42:48.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:48.793486 17785 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
23:42:48.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:48.808714 17879 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:42:48.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:48.808952 17879 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:42:49.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:49.567966 17129 debug-util.cc:398] Leaking SignalData structure 0x7b08000afc80 after lost signal to thread 16998
23:42:49.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:49.569007 17129 debug-util.cc:398] Leaking SignalData structure 0x7b08000b9bc0 after lost signal to thread 17132
23:42:49.765 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.227.253:34331(127.16.227.253:34331)] connecting to peer
23:42:49.767 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.227.254:42395(127.16.227.254:42395)] connecting to peer
23:42:49.767 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:49.768 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.227.253:34331(127.16.227.253:34331)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:49.768 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.227.253:34331: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:49.769 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:49.769 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.227.254:42395(127.16.227.254:42395)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:49.770 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.227.254:42395: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:49.771 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209; will retry
23:42:49.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:49.811844 17879 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:42:49.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:49.812211 17879 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:42:50.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:50.815061 17879 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:42:50.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:50.815312 17879 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:42:51.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:51.822134 17879 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:42:51.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:51.822522 17879 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:42:52.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:52.826072 17879 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:42:52.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:52.826478 17879 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:42:52.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:52.857681 17858 debug-util.cc:398] Leaking SignalData structure 0x7b08000b5ba0 after lost signal to thread 17786
23:42:52.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:52.858657 17858 debug-util.cc:398] Leaking SignalData structure 0x7b080007c5e0 after lost signal to thread 17861
23:42:53.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 23:42:53 dist-test-slave-8w9v krb5kdc[17308](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786318973, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.227.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
23:42:53.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:53.424343 17801 init.cc:295] Successfully reacquired a new kerberos TGT
23:42:53.545 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.227.253:34331(127.16.227.253:34331)] connecting to peer
23:42:53.547 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.227.254:42395(127.16.227.254:42395)] connecting to peer
23:42:53.549 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:53.549 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.227.253:34331(127.16.227.253:34331)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:53.549 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:53.549 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.227.253:34331: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:53.549 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.227.254:42395(127.16.227.254:42395)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:53.550 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.227.254:42395: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:53.554 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209; will retry
23:42:53.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:53.823357 17785 master_runner.cc:186] Error getting master registration for 127.16.227.254:42395: Network error: Client connection negotiation failed: client connection to 127.16.227.254:42395: connect: Connection refused (error 111),
23:42:53.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:53.823968 17785 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
23:42:53.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:53.829401 17879 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:42:53.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:53.829648 17879 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:42:54.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:54.832470 17879 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:42:54.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:54.832757 17879 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:42:54.925 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.227.253:34331(127.16.227.253:34331)] connecting to peer
23:42:54.927 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.227.254:42395(127.16.227.254:42395)] connecting to peer
23:42:54.928 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:54.928 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.227.253:34331(127.16.227.253:34331)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:54.928 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.227.253:34331: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:54.929 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:54.929 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.227.254:42395(127.16.227.254:42395)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:54.929 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.227.254:42395: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:54.931 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209; will retry
23:42:55.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:55.045607 17936 raft_consensus.cc:482] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
23:42:55.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:55.046027 17936 raft_consensus.cc:504] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aa1cf38e72e643c39ce903ca13ecdd81" member_type: VOTER last_known_addr { host: "127.16.227.254" port: 42395 } } peers { permanent_uuid: "8fdb81d4101b4b3bb72f6e2f36f173c5" member_type: VOTER last_known_addr { host: "127.16.227.253" port: 34331 } } peers { permanent_uuid: "1bdaf4a71b89405793c1b7bcb04cd5df" member_type: VOTER last_known_addr { host: "127.16.227.252" port: 34209 } }
23:42:55.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:55.047777 17936 leader_election.cc:288] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers aa1cf38e72e643c39ce903ca13ecdd81 (127.16.227.254:42395), 8fdb81d4101b4b3bb72f6e2f36f173c5 (127.16.227.253:34331)
23:42:55.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:55.050087 17807 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.227.254:42395: connect: Connection refused (error 111) [suppressed 3 similar messages]
23:42:55.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:55.053184 17807 leader_election.cc:334] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer aa1cf38e72e643c39ce903ca13ecdd81 (127.16.227.254:42395): Network error: Client connection negotiation failed: client connection to 127.16.227.254:42395: connect: Connection refused (error 111)
23:42:55.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:55.053795 17810 leader_election.cc:334] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 8fdb81d4101b4b3bb72f6e2f36f173c5 (127.16.227.253:34331): Network error: Client connection negotiation failed: client connection to 127.16.227.253:34331: connect: Connection refused (error 111)
23:42:55.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:55.054162 17810 leader_election.cc:302] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1bdaf4a71b89405793c1b7bcb04cd5df; no voters: 8fdb81d4101b4b3bb72f6e2f36f173c5, aa1cf38e72e643c39ce903ca13ecdd81
23:42:55.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:55.054837 17936 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
23:42:55.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:55.835711 17879 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:42:55.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:55.835951 17879 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:42:55.845 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.227.253:34331(127.16.227.253:34331)] connecting to peer
23:42:55.846 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.227.254:42395(127.16.227.254:42395)] connecting to peer
23:42:55.846 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:55.847 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.227.253:34331(127.16.227.253:34331)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:55.847 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.227.253:34331: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:55.848 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:55.848 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.227.254:42395(127.16.227.254:42395)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:55.848 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.227.254:42395: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:55.850 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209; will retry
23:42:56.345 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.227.253:34331(127.16.227.253:34331)] connecting to peer
23:42:56.346 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.227.254:42395(127.16.227.254:42395)] connecting to peer
23:42:56.346 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:56.347 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.227.253:34331(127.16.227.253:34331)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:56.347 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.227.253:34331: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:56.348 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:56.348 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.227.254:42395(127.16.227.254:42395)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:56.348 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.227.254:42395: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:56.350 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209; will retry
23:42:56.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:56.839218 17879 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:42:56.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:56.839462 17879 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:42:57.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:57.842159 17879 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:42:57.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:57.842415 17879 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:42:58.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:58.826745 17785 master_runner.cc:186] Error getting master registration for 127.16.227.254:42395: Network error: Client connection negotiation failed: client connection to 127.16.227.254:42395: connect: Connection refused (error 111),
23:42:58.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:42:58.827126 17785 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
23:42:58.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:58.845077 17879 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:42:58.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:58.845291 17879 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:42:59.665 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.227.253:34331(127.16.227.253:34331)] connecting to peer
23:42:59.666 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.227.254:42395(127.16.227.254:42395)] connecting to peer
23:42:59.666 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:59.667 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.227.253:34331(127.16.227.253:34331)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:59.667 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.227.253:34331: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:42:59.667 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:59.668 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.227.254:42395(127.16.227.254:42395)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:59.668 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.227.254:42395: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:42:59.670 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209; will retry
23:42:59.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:59.847885 17879 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:42:59.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:42:59.848181 17879 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:43:00.305 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.227.253:34331(127.16.227.253:34331)] connecting to peer
23:43:00.306 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.227.254:42395(127.16.227.254:42395)] connecting to peer
23:43:00.306 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:43:00.306 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.227.253:34331(127.16.227.253:34331)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:43:00.307 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.227.253:34331: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:43:00.308 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:43:00.308 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.227.254:42395(127.16.227.254:42395)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:43:00.308 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.227.254:42395: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:43:00.310 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209; will retry
23:43:00.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:43:00.851197 17879 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1bdaf4a71b89405793c1b7bcb04cd5df: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:43:00.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:43:00.851452 17879 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:43:01.305 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.227.253:34331(127.16.227.253:34331)] connecting to peer
23:43:01.305 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.227.254:42395(127.16.227.254:42395)] connecting to peer
23:43:01.306 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:43:01.306 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.227.253:34331(127.16.227.253:34331)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:43:01.306 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.227.253:34331: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331
23:43:01.307 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:43:01.307 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.227.254:42395(127.16.227.254:42395)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:43:01.307 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.227.254:42395: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
23:43:01.309 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209; will retry
23:43:01.314 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=27726), Traces: [0ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.253:34331, [149ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.254:42395, [150ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.252:34209, [216ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.254:42395: Network error: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [219ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.253:34331: Network error: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331, [600ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.252:34209: OK, [606ms] delaying RPC due to: Service unavailable: Master config (127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331, [616ms] refreshing cache from master, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.253:34331, [620ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.253:34331: Network error: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331, [622ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.254:42395, [623ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.252:34209, [627ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.254:42395: Network error: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [629ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.252:34209: OK, [630ms] delaying RPC due to: Service unavailable: Master config (127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [635ms] refreshing cache from master, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.253:34331, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.254:42395, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.252:34209, [641ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.253:34331: Network error: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331, [642ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.254:42395: Network error: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [644ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.252:34209: OK, [645ms] delaying RPC due to: Service unavailable: Master config (127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [656ms] refreshing cache from master, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.253:34331, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.254:42395, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.252:34209, [659ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.253:34331: Network error: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331, [661ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.254:42395: Network error: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [664ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.252:34209: OK, [665ms] delaying RPC due to: Service unavailable: Master config (127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [696ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.253:34331, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.254:42395, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.252:34209, [699ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.253:34331: Network error: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331, [702ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.254:42395: Network error: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [704ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.252:34209: OK, [705ms] delaying RPC due to: Service unavailable: Master config (127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [736ms] refreshing cache from master, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.253:34331, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.254:42395, [739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.252:34209, [740ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.253:34331: Network error: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331, [744ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.254:42395: Network error: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [745ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.252:34209: OK, [746ms] delaying RPC due to: Service unavailable: Master config (127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [796ms] refreshing cache from master, [797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.253:34331, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.254:42395, [799ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.252:34209, [800ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.253:34331: Network error: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331, [803ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.254:42395: Network error: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [805ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.252:34209: OK, [805ms] delaying RPC due to: Service unavailable: Master config (127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [915ms] refreshing cache from master, [916ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.253:34331, [918ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.254:42395, [919ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.252:34209, [919ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.253:34331: Network error: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331, [921ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.254:42395: Network error: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [923ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.252:34209: OK, [924ms] delaying RPC due to: Service unavailable: Master config (127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [955ms] refreshing cache from master, [956ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.253:34331, [957ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.254:42395, [957ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.252:34209, [967ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.253:34331: Network error: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331, [978ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.252:34209: OK, [984ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.254:42395: Network error: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [985ms] delaying RPC due to: Service unavailable: Master config (127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [1316ms] refreshing cache from master, [1316ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.253:34331, [1318ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.254:42395, [1320ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.252:34209, [1320ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.253:34331: Network error: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331, [1322ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.254:42395: Network error: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [1324ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.252:34209: OK, [1325ms] delaying RPC due to: Service unavailable: Master config (127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [2276ms] refreshing cache from master, [2276ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.253:34331, [2277ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.254:42395, [2279ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.252:34209, [2279ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.253:34331: Network error: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331, [2280ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.254:42395: Network error: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [2282ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.252:34209: OK, [2283ms] delaying RPC due to: Service unavailable: Master config (127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [3755ms] refreshing cache from master, [3756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.253:34331, [3757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.254:42395, [3758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.252:34209, [3759ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.253:34331: Network error: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331, [3759ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.254:42395: Network error: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [3763ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.252:34209: OK, [3764ms] delaying RPC due to: Service unavailable: Master config (127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [6236ms] refreshing cache from master, [6236ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.253:34331, [6237ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.254:42395, [6238ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.252:34209, [6239ms] too many traces: truncated at 100 traces, deferred=Deferred@1813428804(state=PENDING, result=null, callback=(continuation of Deferred@2138588352 after retry RPC after error@1921798490) -> (continuation of Deferred@1385902352 after retry RPC after error@193357213) -> (continuation of Deferred@912617065 after retry RPC after error@653520766) -> (continuation of Deferred@762009857 after retry RPC after error@360283068) -> (continuation of Deferred@141711708 after retry RPC after error@834920243) -> (continuation of Deferred@1175927643 after retry RPC after error@418192574) -> (continuation of Deferred@38735430 after retry RPC after error@769287035) -> (continuation of Deferred@1382566008 after retry RPC after error@579905127) -> (continuation of Deferred@1482142029 after retry RPC after error@644399239) -> (continuation of Deferred@911439802 after retry RPC after error@299326601) -> (continuation of Deferred@1961839143 after retry RPC after error@342085829) -> (continuation of Deferred@966822740 after retry RPC after error@314096637) -> (continuation of Deferred@470312447 after retry RPC after error@375876419) -> (continuation of Deferred@723564784 after retry RPC after error@2081462586) -> (continuation of Deferred@602422490 after retry RPC after error@251164311) -> (continuation of Deferred@548513920 after retry RPC after error@1477455077) -> (continuation of Deferred@981259729 after retry RPC after error@2013085895) -> (continuation of Deferred@1512379152 after retry RPC after error@51080117) -> (continuation of Deferred@1602610264 after retry RPC after error@1122200599) -> (continuation of Deferred@808213192 after retry RPC after error@1747114427) -> (continuation of Deferred@418727875 after retry RPC after error@1101988638) -> (continuation of Deferred@327674873 after retry RPC after error@1838552883) -> (continuation of Deferred@1601146397 after retry RPC after error@1726071069) -> (continuation of Deferred@1251292699 after retry RPC after error@392663029), errback=(continuation of Deferred@2138588352 after retry RPC after error@1921798490) -> (continuation of Deferred@1385902352 after retry RPC after error@193357213) -> (continuation of Deferred@912617065 after retry RPC after error@653520766) -> (continuation of Deferred@762009857 after retry RPC after error@360283068) -> (continuation of Deferred@141711708 after retry RPC after error@834920243) -> (continuation of Deferred@1175927643 after retry RPC after error@418192574) -> (continuation of Deferred@38735430 after retry RPC after error@769287035) -> (continuation of Deferred@1382566008 after retry RPC after error@579905127) -> (continuation of Deferred@1482142029 after retry RPC after error@644399239) -> (continuation of Deferred@911439802 after retry RPC after error@299326601) -> (continuation of Deferred@1961839143 after retry RPC after error@342085829) -> (continuation of Deferred@966822740 after retry RPC after error@314096637) -> (continuation of Deferred@470312447 after retry RPC after error@375876419) -> (continuation of Deferred@723564784 after retry RPC after error@2081462586) -> (continuation of Deferred@602422490 after retry RPC after error@251164311) -> (continuation of Deferred@548513920 after retry RPC after error@1477455077) -> (continuation of Deferred@981259729 after retry RPC after error@2013085895) -> (continuation of Deferred@1512379152 after retry RPC after error@51080117) -> (continuation of Deferred@1602610264 after retry RPC after error@1122200599) -> (continuation of Deferred@808213192 after retry RPC after error@1747114427) -> (continuation of Deferred@418727875 after retry RPC after error@1101988638) -> (continuation of Deferred@327674873 after retry RPC after error@1838552883) -> (continuation of Deferred@1601146397 after retry RPC after error@1726071069) -> (continuation of Deferred@1251292699 after retry RPC after error@392663029)))
23:43:01.318 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.16.227.254:42395(127.16.227.254:42395)] disconnecting while in state TERMINATED
23:43:01.320 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.16.227.253:34331(127.16.227.253:34331)] disconnecting while in state TERMINATED
23:43:01.320 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.16.227.252:34209(127.16.227.252:34209)] disconnecting while in state READY
23:43:01.324 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
23:43:01.326 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.16.227.252:34209(127.16.227.252:34209)] handling channelInactive
23:43:01.326 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.227.252:34209(127.16.227.252:34209)] cleaning up while in state READY due to: connection closed
23:43:01.328 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
23:43:01.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:43:01.338883 17295 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskkf9L9h/build/tsan/bin/kudu with pid 17785
23:43:02.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-09T23:43:02Z chronyd exiting
23:43:03.547 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
23:43:03.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:43:03.548738 16520 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskkf9L9h/build/tsan/bin/kudu with pid 16818
23:43:03.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:43:03.580982 16520 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskkf9L9h/build/tsan/bin/kudu with pid 16992
23:43:03.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:43:03.611931 16520 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskkf9L9h/build/tsan/bin/kudu with pid 17143
23:43:03.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:43:03.641263 16520 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskkf9L9h/build/tsan/bin/kudu with pid 16534
23:43:03.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:43:03.677245 16520 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskkf9L9h/build/tsan/bin/kudu with pid 16607
23:43:03.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:43:03.710968 16520 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskkf9L9h/build/tsan/bin/kudu with pid 16688
23:43:04.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-09T23:43:04Z chronyd exiting
23:43:05.857 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=27726), Traces: [0ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.253:34331, [149ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.254:42395, [150ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.252:34209, [216ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.254:42395: Network error: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [219ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.253:34331: Network error: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331, [600ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.252:34209: OK, [606ms] delaying RPC due to: Service unavailable: Master config (127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331, [616ms] refreshing cache from master, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.253:34331, [620ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.253:34331: Network error: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331, [622ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.254:42395, [623ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.252:34209, [627ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.254:42395: Network error: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [629ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.252:34209: OK, [630ms] delaying RPC due to: Service unavailable: Master config (127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [635ms] refreshing cache from master, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.253:34331, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.254:42395, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.252:34209, [641ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.253:34331: Network error: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331, [642ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.254:42395: Network error: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [644ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.252:34209: OK, [645ms] delaying RPC due to: Service unavailable: Master config (127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [656ms] refreshing cache from master, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.253:34331, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.254:42395, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.252:34209, [659ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.253:34331: Network error: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331, [661ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.254:42395: Network error: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [664ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.252:34209: OK, [665ms] delaying RPC due to: Service unavailable: Master config (127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [696ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.253:34331, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.254:42395, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.252:34209, [699ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.253:34331: Network error: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331, [702ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.254:42395: Network error: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [704ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.252:34209: OK, [705ms] delaying RPC due to: Service unavailable: Master config (127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [736ms] refreshing cache from master, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.253:34331, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.254:42395, [739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.252:34209, [740ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.253:34331: Network error: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331, [744ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.254:42395: Network error: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [745ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.252:34209: OK, [746ms] delaying RPC due to: Service unavailable: Master config (127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [796ms] refreshing cache from master, [797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.253:34331, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.254:42395, [799ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.252:34209, [800ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.253:34331: Network error: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331, [803ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.254:42395: Network error: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [805ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.252:34209: OK, [805ms] delaying RPC due to: Service unavailable: Master config (127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [915ms] refreshing cache from master, [916ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.253:34331, [918ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.254:42395, [919ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.252:34209, [919ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.253:34331: Network error: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331, [921ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.254:42395: Network error: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [923ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.252:34209: OK, [924ms] delaying RPC due to: Service unavailable: Master config (127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [955ms] refreshing cache from master, [956ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.253:34331, [957ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.254:42395, [957ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.252:34209, [967ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.253:34331: Network error: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331, [978ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.252:34209: OK, [984ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.254:42395: Network error: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [985ms] delaying RPC due to: Service unavailable: Master config (127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [1316ms] refreshing cache from master, [1316ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.253:34331, [1318ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.254:42395, [1320ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.252:34209, [1320ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.253:34331: Network error: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331, [1322ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.254:42395: Network error: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [1324ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.252:34209: OK, [1325ms] delaying RPC due to: Service unavailable: Master config (127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [2276ms] refreshing cache from master, [2276ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.253:34331, [2277ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.254:42395, [2279ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.252:34209, [2279ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.253:34331: Network error: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331, [2280ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.254:42395: Network error: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [2282ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.252:34209: OK, [2283ms] delaying RPC due to: Service unavailable: Master config (127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [3755ms] refreshing cache from master, [3756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.253:34331, [3757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.254:42395, [3758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.252:34209, [3759ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.253:34331: Network error: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331, [3759ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.254:42395: Network error: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [3763ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.252:34209: OK, [3764ms] delaying RPC due to: Service unavailable: Master config (127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [6236ms] refreshing cache from master, [6236ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.253:34331, [6237ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.254:42395, [6238ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.252:34209, [6239ms] too many traces: truncated at 100 traces, deferred=Deferred@1813428804(state=PENDING, result=null, callback=(continuation of Deferred@2138588352 after retry RPC after error@1921798490) -> (continuation of Deferred@1385902352 after retry RPC after error@193357213) -> (continuation of Deferred@912617065 after retry RPC after error@653520766) -> (continuation of Deferred@762009857 after retry RPC after error@360283068) -> (continuation of Deferred@141711708 after retry RPC after error@834920243) -> (continuation of Deferred@1175927643 after retry RPC after error@418192574) -> (continuation of Deferred@38735430 after retry RPC after error@769287035) -> (continuation of Deferred@1382566008 after retry RPC after error@579905127) -> (continuation of Deferred@1482142029 after retry RPC after error@644399239) -> (continuation of Deferred@911439802 after retry RPC after error@299326601) -> (continuation of Deferred@1961839143 after retry RPC after error@342085829) -> (continuation of Deferred@966822740 after retry RPC after error@314096637) -> (continuation of Deferred@470312447 after retry RPC after error@375876419) -> (continuation of Deferred@723564784 after retry RPC after error@2081462586) -> (continuation of Deferred@602422490 after retry RPC after error@251164311) -> (continuation of Deferred@548513920 after retry RPC after error@1477455077) -> (continuation of Deferred@981259729 after retry RPC after error@2013085895) -> (continuation of Deferred@1512379152 after retry RPC after error@51080117) -> (continuation of Deferred@1602610264 after retry RPC after error@1122200599) -> (continuation of Deferred@808213192 after retry RPC after error@1747114427) -> (continuation of Deferred@418727875 after retry RPC after error@1101988638) -> (continuation of Deferred@327674873 after retry RPC after error@1838552883) -> (continuation of Deferred@1601146397 after retry RPC after error@1726071069) -> (continuation of Deferred@1251292699 after retry RPC after error@392663029), errback=(continuation of Deferred@2138588352 after retry RPC after error@1921798490) -> (continuation of Deferred@1385902352 after retry RPC after error@193357213) -> (continuation of Deferred@912617065 after retry RPC after error@653520766) -> (continuation of Deferred@762009857 after retry RPC after error@360283068) -> (continuation of Deferred@141711708 after retry RPC after error@834920243) -> (continuation of Deferred@1175927643 after retry RPC after error@418192574) -> (continuation of Deferred@38735430 after retry RPC after error@769287035) -> (continuation of Deferred@1382566008 after retry RPC after error@579905127) -> (continuation of Deferred@1482142029 after retry RPC after error@644399239) -> (continuation of Deferred@911439802 after retry RPC after error@299326601) -> (continuation of Deferred@1961839143 after retry RPC after error@342085829) -> (continuation of Deferred@966822740 after retry RPC after error@314096637) -> (continuation of Deferred@470312447 after retry RPC after error@375876419) -> (continuation of Deferred@723564784 after retry RPC after error@2081462586) -> (continuation of Deferred@602422490 after retry RPC after error@251164311) -> (continuation of Deferred@548513920 after retry RPC after error@1477455077) -> (continuation of Deferred@981259729 after retry RPC after error@2013085895) -> (continuation of Deferred@1512379152 after retry RPC after error@51080117) -> (continuation of Deferred@1602610264 after retry RPC after error@1122200599) -> (continuation of Deferred@808213192 after retry RPC after error@1747114427) -> (continuation of Deferred@418727875 after retry RPC after error@1101988638) -> (continuation of Deferred@327674873 after retry RPC after error@1838552883) -> (continuation of Deferred@1601146397 after retry RPC after error@1726071069) -> (continuation of Deferred@1251292699 after retry RPC after error@392663029)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.16.227.254:42395
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=27726), Traces: [0ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.253:34331, [149ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.254:42395, [150ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.252:34209, [216ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.254:42395: Network error: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [219ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.253:34331: Network error: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331, [600ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.252:34209: OK, [606ms] delaying RPC due to: Service unavailable: Master config (127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331, [616ms] refreshing cache from master, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.253:34331, [620ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.253:34331: Network error: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331, [622ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.254:42395, [623ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.252:34209, [627ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.254:42395: Network error: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [629ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.252:34209: OK, [630ms] delaying RPC due to: Service unavailable: Master config (127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [635ms] refreshing cache from master, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.253:34331, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.254:42395, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.252:34209, [641ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.253:34331: Network error: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331, [642ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.254:42395: Network error: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [644ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.252:34209: OK, [645ms] delaying RPC due to: Service unavailable: Master config (127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [656ms] refreshing cache from master, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.253:34331, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.254:42395, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.252:34209, [659ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.253:34331: Network error: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331, [661ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.254:42395: Network error: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [664ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.252:34209: OK, [665ms] delaying RPC due to: Service unavailable: Master config (127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [696ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.253:34331, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.254:42395, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.252:34209, [699ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.253:34331: Network error: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331, [702ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.254:42395: Network error: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [704ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.252:34209: OK, [705ms] delaying RPC due to: Service unavailable: Master config (127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [736ms] refreshing cache from master, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.253:34331, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.254:42395, [739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.252:34209, [740ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.253:34331: Network error: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331, [744ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.254:42395: Network error: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [745ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.252:34209: OK, [746ms] delaying RPC due to: Service unavailable: Master config (127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [796ms] refreshing cache from master, [797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.253:34331, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.254:42395, [799ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.252:34209, [800ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.253:34331: Network error: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331, [803ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.254:42395: Network error: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [805ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.252:34209: OK, [805ms] delaying RPC due to: Service unavailable: Master config (127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [915ms] refreshing cache from master, [916ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.253:34331, [918ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.254:42395, [919ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.252:34209, [919ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.253:34331: Network error: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331, [921ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.254:42395: Network error: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [923ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.252:34209: OK, [924ms] delaying RPC due to: Service unavailable: Master config (127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [955ms] refreshing cache from master, [956ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.253:34331, [957ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.254:42395, [957ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.252:34209, [967ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.253:34331: Network error: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331, [978ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.252:34209: OK, [984ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.254:42395: Network error: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [985ms] delaying RPC due to: Service unavailable: Master config (127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [1316ms] refreshing cache from master, [1316ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.253:34331, [1318ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.254:42395, [1320ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.252:34209, [1320ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.253:34331: Network error: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331, [1322ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.254:42395: Network error: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [1324ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.252:34209: OK, [1325ms] delaying RPC due to: Service unavailable: Master config (127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [2276ms] refreshing cache from master, [2276ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.253:34331, [2277ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.254:42395, [2279ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.252:34209, [2279ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.253:34331: Network error: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331, [2280ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.254:42395: Network error: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [2282ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.252:34209: OK, [2283ms] delaying RPC due to: Service unavailable: Master config (127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [3755ms] refreshing cache from master, [3756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.253:34331, [3757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.254:42395, [3758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.252:34209, [3759ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.253:34331: Network error: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331, [3759ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.254:42395: Network error: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [3763ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.252:34209: OK, [3764ms] delaying RPC due to: Service unavailable: Master config (127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [6236ms] refreshing cache from master, [6236ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.253:34331, [6237ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.254:42395, [6238ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.252:34209, [6239ms] too many traces: truncated at 100 traces, deferred=Deferred@1813428804(state=PENDING, result=null, callback=(continuation of Deferred@2138588352 after retry RPC after error@1921798490) -> (continuation of Deferred@1385902352 after retry RPC after error@193357213) -> (continuation of Deferred@912617065 after retry RPC after error@653520766) -> (continuation of Deferred@762009857 after retry RPC after error@360283068) -> (continuation of Deferred@141711708 after retry RPC after error@834920243) -> (continuation of Deferred@1175927643 after retry RPC after error@418192574) -> (continuation of Deferred@38735430 after retry RPC after error@769287035) -> (continuation of Deferred@1382566008 after retry RPC after error@579905127) -> (continuation of Deferred@1482142029 after retry RPC after error@644399239) -> (continuation of Deferred@911439802 after retry RPC after error@299326601) -> (continuation of Deferred@1961839143 after retry RPC after error@342085829) -> (continuation of Deferred@966822740 after retry RPC after error@314096637) -> (continuation of Deferred@470312447 after retry RPC after error@375876419) -> (continuation of Deferred@723564784 after retry RPC after error@2081462586) -> (continuation of Deferred@602422490 after retry RPC after error@251164311) -> (continuation of Deferred@548513920 after retry RPC after error@1477455077) -> (continuation of Deferred@981259729 after retry RPC after error@2013085895) -> (continuation of Deferred@1512379152 after retry RPC after error@51080117) -> (continuation of Deferred@1602610264 after retry RPC after error@1122200599) -> (continuation of Deferred@808213192 after retry RPC after error@1747114427) -> (continuation of Deferred@418727875 after retry RPC after error@1101988638) -> (continuation of Deferred@327674873 after retry RPC after error@1838552883) -> (continuation of Deferred@1601146397 after retry RPC after error@1726071069) -> (continuation of Deferred@1251292699 after retry RPC after error@392663029), errback=(continuation of Deferred@2138588352 after retry RPC after error@1921798490) -> (continuation of Deferred@1385902352 after retry RPC after error@193357213) -> (continuation of Deferred@912617065 after retry RPC after error@653520766) -> (continuation of Deferred@762009857 after retry RPC after error@360283068) -> (continuation of Deferred@141711708 after retry RPC after error@834920243) -> (continuation of Deferred@1175927643 after retry RPC after error@418192574) -> (continuation of Deferred@38735430 after retry RPC after error@769287035) -> (continuation of Deferred@1382566008 after retry RPC after error@579905127) -> (continuation of Deferred@1482142029 after retry RPC after error@644399239) -> (continuation of Deferred@911439802 after retry RPC after error@299326601) -> (continuation of Deferred@1961839143 after retry RPC after error@342085829) -> (continuation of Deferred@966822740 after retry RPC after error@314096637) -> (continuation of Deferred@470312447 after retry RPC after error@375876419) -> (continuation of Deferred@723564784 after retry RPC after error@2081462586) -> (continuation of Deferred@602422490 after retry RPC after error@251164311) -> (continuation of Deferred@548513920 after retry RPC after error@1477455077) -> (continuation of Deferred@981259729 after retry RPC after error@2013085895) -> (continuation of Deferred@1512379152 after retry RPC after error@51080117) -> (continuation of Deferred@1602610264 after retry RPC after error@1122200599) -> (continuation of Deferred@808213192 after retry RPC after error@1747114427) -> (continuation of Deferred@418727875 after retry RPC after error@1101988638) -> (continuation of Deferred@327674873 after retry RPC after error@1838552883) -> (continuation of Deferred@1601146397 after retry RPC after error@1726071069) -> (continuation of Deferred@1251292699 after retry RPC after error@392663029)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.16.227.254:42395
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
23:43:06.319 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
23:43:06.319 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=27726), Traces: [0ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.253:34331, [149ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.254:42395, [150ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.252:34209, [216ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.254:42395: Network error: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [219ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.253:34331: Network error: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331, [600ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.252:34209: OK, [606ms] delaying RPC due to: Service unavailable: Master config (127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331, [616ms] refreshing cache from master, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.253:34331, [620ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.253:34331: Network error: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331, [622ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.254:42395, [623ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.252:34209, [627ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.254:42395: Network error: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [629ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.252:34209: OK, [630ms] delaying RPC due to: Service unavailable: Master config (127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [635ms] refreshing cache from master, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.253:34331, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.254:42395, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.252:34209, [641ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.253:34331: Network error: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331, [642ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.254:42395: Network error: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [644ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.252:34209: OK, [645ms] delaying RPC due to: Service unavailable: Master config (127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [656ms] refreshing cache from master, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.253:34331, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.254:42395, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.252:34209, [659ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.253:34331: Network error: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331, [661ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.254:42395: Network error: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [664ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.252:34209: OK, [665ms] delaying RPC due to: Service unavailable: Master config (127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [696ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.253:34331, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.254:42395, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.252:34209, [699ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.253:34331: Network error: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331, [702ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.254:42395: Network error: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [704ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.252:34209: OK, [705ms] delaying RPC due to: Service unavailable: Master config (127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [736ms] refreshing cache from master, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.253:34331, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.254:42395, [739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.252:34209, [740ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.253:34331: Network error: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331, [744ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.254:42395: Network error: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [745ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.252:34209: OK, [746ms] delaying RPC due to: Service unavailable: Master config (127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [796ms] refreshing cache from master, [797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.253:34331, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.254:42395, [799ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.252:34209, [800ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.253:34331: Network error: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331, [803ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.254:42395: Network error: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [805ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.252:34209: OK, [805ms] delaying RPC due to: Service unavailable: Master config (127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [915ms] refreshing cache from master, [916ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.253:34331, [918ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.254:42395, [919ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.252:34209, [919ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.253:34331: Network error: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331, [921ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.254:42395: Network error: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [923ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.252:34209: OK, [924ms] delaying RPC due to: Service unavailable: Master config (127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [955ms] refreshing cache from master, [956ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.253:34331, [957ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.254:42395, [957ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.252:34209, [967ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.253:34331: Network error: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331, [978ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.252:34209: OK, [984ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.254:42395: Network error: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [985ms] delaying RPC due to: Service unavailable: Master config (127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [1316ms] refreshing cache from master, [1316ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.253:34331, [1318ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.254:42395, [1320ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.252:34209, [1320ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.253:34331: Network error: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331, [1322ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.254:42395: Network error: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [1324ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.252:34209: OK, [1325ms] delaying RPC due to: Service unavailable: Master config (127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [2276ms] refreshing cache from master, [2276ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.253:34331, [2277ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.254:42395, [2279ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.252:34209, [2279ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.253:34331: Network error: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331, [2280ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.254:42395: Network error: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [2282ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.252:34209: OK, [2283ms] delaying RPC due to: Service unavailable: Master config (127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [3755ms] refreshing cache from master, [3756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.253:34331, [3757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.254:42395, [3758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.252:34209, [3759ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.253:34331: Network error: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331, [3759ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.254:42395: Network error: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [3763ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.252:34209: OK, [3764ms] delaying RPC due to: Service unavailable: Master config (127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [6236ms] refreshing cache from master, [6236ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.253:34331, [6237ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.254:42395, [6238ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.252:34209, [6239ms] too many traces: truncated at 100 traces, deferred=Deferred@1813428804(state=PENDING, result=null, callback=(continuation of Deferred@2138588352 after retry RPC after error@1921798490) -> (continuation of Deferred@1385902352 after retry RPC after error@193357213) -> (continuation of Deferred@912617065 after retry RPC after error@653520766) -> (continuation of Deferred@762009857 after retry RPC after error@360283068) -> (continuation of Deferred@141711708 after retry RPC after error@834920243) -> (continuation of Deferred@1175927643 after retry RPC after error@418192574) -> (continuation of Deferred@38735430 after retry RPC after error@769287035) -> (continuation of Deferred@1382566008 after retry RPC after error@579905127) -> (continuation of Deferred@1482142029 after retry RPC after error@644399239) -> (continuation of Deferred@911439802 after retry RPC after error@299326601) -> (continuation of Deferred@1961839143 after retry RPC after error@342085829) -> (continuation of Deferred@966822740 after retry RPC after error@314096637) -> (continuation of Deferred@470312447 after retry RPC after error@375876419) -> (continuation of Deferred@723564784 after retry RPC after error@2081462586) -> (continuation of Deferred@602422490 after retry RPC after error@251164311) -> (continuation of Deferred@548513920 after retry RPC after error@1477455077) -> (continuation of Deferred@981259729 after retry RPC after error@2013085895) -> (continuation of Deferred@1512379152 after retry RPC after error@51080117) -> (continuation of Deferred@1602610264 after retry RPC after error@1122200599) -> (continuation of Deferred@808213192 after retry RPC after error@1747114427) -> (continuation of Deferred@418727875 after retry RPC after error@1101988638) -> (continuation of Deferred@327674873 after retry RPC after error@1838552883) -> (continuation of Deferred@1601146397 after retry RPC after error@1726071069) -> (continuation of Deferred@1251292699 after retry RPC after error@392663029), errback=(continuation of Deferred@2138588352 after retry RPC after error@1921798490) -> (continuation of Deferred@1385902352 after retry RPC after error@193357213) -> (continuation of Deferred@912617065 after retry RPC after error@653520766) -> (continuation of Deferred@762009857 after retry RPC after error@360283068) -> (continuation of Deferred@141711708 after retry RPC after error@834920243) -> (continuation of Deferred@1175927643 after retry RPC after error@418192574) -> (continuation of Deferred@38735430 after retry RPC after error@769287035) -> (continuation of Deferred@1382566008 after retry RPC after error@579905127) -> (continuation of Deferred@1482142029 after retry RPC after error@644399239) -> (continuation of Deferred@911439802 after retry RPC after error@299326601) -> (continuation of Deferred@1961839143 after retry RPC after error@342085829) -> (continuation of Deferred@966822740 after retry RPC after error@314096637) -> (continuation of Deferred@470312447 after retry RPC after error@375876419) -> (continuation of Deferred@723564784 after retry RPC after error@2081462586) -> (continuation of Deferred@602422490 after retry RPC after error@251164311) -> (continuation of Deferred@548513920 after retry RPC after error@1477455077) -> (continuation of Deferred@981259729 after retry RPC after error@2013085895) -> (continuation of Deferred@1512379152 after retry RPC after error@51080117) -> (continuation of Deferred@1602610264 after retry RPC after error@1122200599) -> (continuation of Deferred@808213192 after retry RPC after error@1747114427) -> (continuation of Deferred@418727875 after retry RPC after error@1101988638) -> (continuation of Deferred@327674873 after retry RPC after error@1838552883) -> (continuation of Deferred@1601146397 after retry RPC after error@1726071069) -> (continuation of Deferred@1251292699 after retry RPC after error@392663029)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.16.227.254:42395
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=27726), Traces: [0ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.253:34331, [149ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.254:42395, [150ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.252:34209, [216ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.254:42395: Network error: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [219ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.253:34331: Network error: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331, [600ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.252:34209: OK, [606ms] delaying RPC due to: Service unavailable: Master config (127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331, [616ms] refreshing cache from master, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.253:34331, [620ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.253:34331: Network error: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331, [622ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.254:42395, [623ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.252:34209, [627ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.254:42395: Network error: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [629ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.252:34209: OK, [630ms] delaying RPC due to: Service unavailable: Master config (127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [635ms] refreshing cache from master, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.253:34331, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.254:42395, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.252:34209, [641ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.253:34331: Network error: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331, [642ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.254:42395: Network error: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [644ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.252:34209: OK, [645ms] delaying RPC due to: Service unavailable: Master config (127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [656ms] refreshing cache from master, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.253:34331, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.254:42395, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.252:34209, [659ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.253:34331: Network error: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331, [661ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.254:42395: Network error: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [664ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.252:34209: OK, [665ms] delaying RPC due to: Service unavailable: Master config (127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [696ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.253:34331, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.254:42395, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.252:34209, [699ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.253:34331: Network error: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331, [702ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.254:42395: Network error: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [704ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.252:34209: OK, [705ms] delaying RPC due to: Service unavailable: Master config (127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [736ms] refreshing cache from master, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.253:34331, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.254:42395, [739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.252:34209, [740ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.253:34331: Network error: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331, [744ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.254:42395: Network error: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [745ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.252:34209: OK, [746ms] delaying RPC due to: Service unavailable: Master config (127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [796ms] refreshing cache from master, [797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.253:34331, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.254:42395, [799ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.252:34209, [800ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.253:34331: Network error: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331, [803ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.254:42395: Network error: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [805ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.252:34209: OK, [805ms] delaying RPC due to: Service unavailable: Master config (127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [915ms] refreshing cache from master, [916ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.253:34331, [918ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.254:42395, [919ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.252:34209, [919ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.253:34331: Network error: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331, [921ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.254:42395: Network error: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [923ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.252:34209: OK, [924ms] delaying RPC due to: Service unavailable: Master config (127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [955ms] refreshing cache from master, [956ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.253:34331, [957ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.254:42395, [957ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.252:34209, [967ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.253:34331: Network error: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331, [978ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.252:34209: OK, [984ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.254:42395: Network error: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [985ms] delaying RPC due to: Service unavailable: Master config (127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [1316ms] refreshing cache from master, [1316ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.253:34331, [1318ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.254:42395, [1320ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.252:34209, [1320ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.253:34331: Network error: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331, [1322ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.254:42395: Network error: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [1324ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.252:34209: OK, [1325ms] delaying RPC due to: Service unavailable: Master config (127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [2276ms] refreshing cache from master, [2276ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.253:34331, [2277ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.254:42395, [2279ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.252:34209, [2279ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.253:34331: Network error: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331, [2280ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.254:42395: Network error: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [2282ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.252:34209: OK, [2283ms] delaying RPC due to: Service unavailable: Master config (127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [3755ms] refreshing cache from master, [3756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.253:34331, [3757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.254:42395, [3758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.252:34209, [3759ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.253:34331: Network error: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331, [3759ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.254:42395: Network error: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [3763ms] Sub RPC ConnectToMaster: received response from server master-127.16.227.252:34209: OK, [3764ms] delaying RPC due to: Service unavailable: Master config (127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395, [6236ms] refreshing cache from master, [6236ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.253:34331, [6237ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.254:42395, [6238ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.227.252:34209, [6239ms] too many traces: truncated at 100 traces, deferred=Deferred@1813428804(state=PENDING, result=null, callback=(continuation of Deferred@2138588352 after retry RPC after error@1921798490) -> (continuation of Deferred@1385902352 after retry RPC after error@193357213) -> (continuation of Deferred@912617065 after retry RPC after error@653520766) -> (continuation of Deferred@762009857 after retry RPC after error@360283068) -> (continuation of Deferred@141711708 after retry RPC after error@834920243) -> (continuation of Deferred@1175927643 after retry RPC after error@418192574) -> (continuation of Deferred@38735430 after retry RPC after error@769287035) -> (continuation of Deferred@1382566008 after retry RPC after error@579905127) -> (continuation of Deferred@1482142029 after retry RPC after error@644399239) -> (continuation of Deferred@911439802 after retry RPC after error@299326601) -> (continuation of Deferred@1961839143 after retry RPC after error@342085829) -> (continuation of Deferred@966822740 after retry RPC after error@314096637) -> (continuation of Deferred@470312447 after retry RPC after error@375876419) -> (continuation of Deferred@723564784 after retry RPC after error@2081462586) -> (continuation of Deferred@602422490 after retry RPC after error@251164311) -> (continuation of Deferred@548513920 after retry RPC after error@1477455077) -> (continuation of Deferred@981259729 after retry RPC after error@2013085895) -> (continuation of Deferred@1512379152 after retry RPC after error@51080117) -> (continuation of Deferred@1602610264 after retry RPC after error@1122200599) -> (continuation of Deferred@808213192 after retry RPC after error@1747114427) -> (continuation of Deferred@418727875 after retry RPC after error@1101988638) -> (continuation of Deferred@327674873 after retry RPC after error@1838552883) -> (continuation of Deferred@1601146397 after retry RPC after error@1726071069) -> (continuation of Deferred@1251292699 after retry RPC after error@392663029), errback=(continuation of Deferred@2138588352 after retry RPC after error@1921798490) -> (continuation of Deferred@1385902352 after retry RPC after error@193357213) -> (continuation of Deferred@912617065 after retry RPC after error@653520766) -> (continuation of Deferred@762009857 after retry RPC after error@360283068) -> (continuation of Deferred@141711708 after retry RPC after error@834920243) -> (continuation of Deferred@1175927643 after retry RPC after error@418192574) -> (continuation of Deferred@38735430 after retry RPC after error@769287035) -> (continuation of Deferred@1382566008 after retry RPC after error@579905127) -> (continuation of Deferred@1482142029 after retry RPC after error@644399239) -> (continuation of Deferred@911439802 after retry RPC after error@299326601) -> (continuation of Deferred@1961839143 after retry RPC after error@342085829) -> (continuation of Deferred@966822740 after retry RPC after error@314096637) -> (continuation of Deferred@470312447 after retry RPC after error@375876419) -> (continuation of Deferred@723564784 after retry RPC after error@2081462586) -> (continuation of Deferred@602422490 after retry RPC after error@251164311) -> (continuation of Deferred@548513920 after retry RPC after error@1477455077) -> (continuation of Deferred@981259729 after retry RPC after error@2013085895) -> (continuation of Deferred@1512379152 after retry RPC after error@51080117) -> (continuation of Deferred@1602610264 after retry RPC after error@1122200599) -> (continuation of Deferred@808213192 after retry RPC after error@1747114427) -> (continuation of Deferred@418727875 after retry RPC after error@1101988638) -> (continuation of Deferred@327674873 after retry RPC after error@1838552883) -> (continuation of Deferred@1601146397 after retry RPC after error@1726071069) -> (continuation of Deferred@1251292699 after retry RPC after error@392663029)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.16.227.253:34331,127.16.227.254:42395,127.16.227.252:34209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.253:34331(127.16.227.253:34331): Connection refused: /127.16.227.253:34331,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.227.254:42395(127.16.227.254:42395): Connection refused: /127.16.227.254:42395
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.16.227.254:42395
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)