23:33:32.724 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
23:33:32.740 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task5zWf6n/test-tmp
23:33:32.743 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
23:33:32.748 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task5zWf6n/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
23:33:32.983 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
23:33:33.318 [DEBUG - main] (MiniKuduCluster.java:195) Response:
23:33:33.319 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
23:33:33.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:33.324136 10389 mini_oidc.cc:152] Starting JWKS server
23:33:33.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:33.330262 10389 webserver.cc:460] Webserver started at http://127.0.0.1:43035/ using document root <none> and password file <none>
23:33:33.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:33.331003 10389 mini_oidc.cc:189] Starting OIDC Discovery server
23:33:33.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:33.333357 10389 webserver.cc:460] Webserver started at http://127.0.0.1:36593/ using document root <none> and password file <none>
23:33:33.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-09T23:33:33Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
23:33:33.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-09T23:33:33Z Disabled control of system clock
23:33:33.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:33.399711 10389 external_mini_cluster.cc:1326] Running /tmp/dist-test-task5zWf6n/build/tsan/bin/kudu
23:33:33.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task5zWf6n/build/tsan/bin/kudu
23:33:33.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/master-0/wal
23:33:33.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/master-0/data
23:33:33.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:33:33.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:33:33.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:33:33.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:33:33.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:33:33.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:33:33.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/master-0/logs
23:33:33.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/master-0/data/info.pb
23:33:33.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:33:33.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:33:33.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:33:33.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:33:33.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:33:33.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:33:33.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:33:33.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:33:33.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
23:33:33.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:33:33.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:33:33.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:33:33.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.37.126:46523
23:33:33.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.37.126
23:33:33.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:33:33.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.37.84:38877
23:33:33.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:33:33.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:33:33.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:33:33.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:33:33.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.37.126:46523,127.10.37.125:37371,127.10.37.124:38379
23:33:33.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:33:33.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43035/jwks.json with env {}
23:33:33.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:33.748684 10404 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:33:33.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:33.749318 10404 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:33:33.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:33.750030 10404 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:33:33.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:33.783577 10404 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
23:33:33.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:33.783890 10404 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:33:33.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:33.784170 10404 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
23:33:33.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:33.784407 10404 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
23:33:33.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:33.784711 10404 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
23:33:33.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:33.784977 10404 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43035/jwks.json
23:33:33.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:33.822526 10404 master_runner.cc:386] Master server non-default flags:
23:33:33.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:33:33.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.37.84:38877
23:33:33.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:33:33.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:33:33.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/master-0/data
23:33:33.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/master-0/wal
23:33:33.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:33:33.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.37.126:46523,127.10.37.125:37371,127.10.37.124:38379
23:33:33.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:33:33.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:33:33.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:33:33.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.37.126:46523
23:33:33.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:33:33.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:33:33.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:33:33.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43035/jwks.json
23:33:33.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:33:33.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:33:33.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/master-0/data/info.pb
23:33:33.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.37.126
23:33:33.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:33:33.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:33:33.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:33:33.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:33:33.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:33:33.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:33:33.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/master-0/logs
23:33:33.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:33:33.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:33:33.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
23:33:33.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
23:33:33.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:33:33.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:33:33.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:33:33.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 20:03:34 UTC on ccycloud.quasar-agiwfo.root.comops.site
23:33:33.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4576543
23:33:33.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:33:33.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:33.824740 10404 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:33:33.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:33.827373 10404 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:33:33.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:33.861709 10410 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:33:35.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:33.861717 10413 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:33:35.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:33.862535 10411 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:33:35.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:33.863667 10404 server_base.cc:1034] running on GCE node
23:33:35.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:35.216205 10404 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:33:35.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:35.219653 10404 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:33:35.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:35.221072 10404 hybrid_clock.cc:648] HybridClock initialized: now 1786318415221015 us; error 97 us; skew 500 ppm
23:33:35.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:35.222337 10404 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:33:35.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:35.229974 10404 webserver.cc:460] Webserver started at http://127.10.37.126:36495/ using document root <none> and password file <none>
23:33:35.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:35.231546 10404 fs_manager.cc:359] Metadata directory not provided
23:33:35.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:35.231812 10404 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:33:35.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:35.232522 10404 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
23:33:35.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:35.238564 10404 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/master-0/data/instance:
23:33:35.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4c6ad93e58f848efa29d1594b7f18918"
23:33:35.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 23:33:35 on dist-test-slave-7c35"
23:33:35.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:35.241567 10404 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/master-0/wal/instance:
23:33:35.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4c6ad93e58f848efa29d1594b7f18918"
23:33:35.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 23:33:35 on dist-test-slave-7c35"
23:33:35.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:35.251950 10404 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.009s sys 0.001s
23:33:35.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:35.259389 10421 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
23:33:35.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:35.261179 10404 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.002s sys 0.001s
23:33:35.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:35.261569 10404 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/master-0/data,/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/master-0/wal
23:33:35.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4c6ad93e58f848efa29d1594b7f18918"
23:33:35.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 23:33:35 on dist-test-slave-7c35"
23:33:35.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:35.262056 10404 fs_report.cc:383] FS layout report
23:33:35.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:33:35.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/master-0/wal
23:33:35.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/master-0/wal
23:33:35.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/master-0/data/data
23:33:35.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:33:35.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:33:35.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:33:35.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:33:35.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:33:35.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:33:35.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:33:35.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:33:35.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:33:35.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:33:35.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:33:35.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:35.305822 10404 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
23:33:35.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:35.307808 10404 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:33:35.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:35.308308 10404 kserver.cc:163] Server-wide thread pool size limit: 3276
23:33:35.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:35.452862 10404 rpc_server.cc:307] RPC server started. Bound to: 127.10.37.126:46523
23:33:35.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:35.453282 10478 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.37.126:46523 every 8 connection(s)
23:33:35.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:35.456758 10404 server_base.cc:1166] Dumped server information to /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/master-0/data/info.pb
23:33:35.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:35.459074 10389 external_mini_cluster.cc:1388] Started /tmp/dist-test-task5zWf6n/build/tsan/bin/kudu as pid 10404
23:33:35.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:35.459594 10389 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/master-0/wal/instance
23:33:35.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:35.465775 10389 external_mini_cluster.cc:1326] Running /tmp/dist-test-task5zWf6n/build/tsan/bin/kudu
23:33:35.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task5zWf6n/build/tsan/bin/kudu
23:33:35.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/master-1/wal
23:33:35.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/master-1/data
23:33:35.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:33:35.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:33:35.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:33:35.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:33:35.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:33:35.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:33:35.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/master-1/logs
23:33:35.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/master-1/data/info.pb
23:33:35.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:33:35.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:33:35.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:33:35.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:33:35.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:33:35.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:33:35.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:33:35.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:33:35.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
23:33:35.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:33:35.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:33:35.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:33:35.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.37.125:37371
23:33:35.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.37.125
23:33:35.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:33:35.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.37.84:38877
23:33:35.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:33:35.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:33:35.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:33:35.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:33:35.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.37.126:46523,127.10.37.125:37371,127.10.37.124:38379
23:33:35.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:33:35.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43035/jwks.json with env {}
23:33:35.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:35.473179 10479 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
23:33:35.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:35.495787 10479 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.37.126" port: 46523 } has no permanent_uuid. Determining permanent_uuid...
23:33:35.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:35.545542 10479 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.37.125" port: 37371 } has no permanent_uuid. Determining permanent_uuid...
23:33:35.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:35.549722 10424 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.37.125:37371: connect: Connection refused (error 111)
23:33:35.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:35.559139 10479 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.37.125:37371: Network error: Client connection negotiation failed: client connection to 127.10.37.125:37371: connect: Connection refused (error 111)
23:33:35.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:35.608728 10479 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.37.125" port: 37371 } attempt: 1
23:33:35.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:35.613652 10479 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.37.125:37371: Network error: Client connection negotiation failed: client connection to 127.10.37.125:37371: connect: Connection refused (error 111)
23:33:35.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:35.682282 10479 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.37.125" port: 37371 } attempt: 2
23:33:35.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:35.687022 10479 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.37.125:37371: Network error: Client connection negotiation failed: client connection to 127.10.37.125:37371: connect: Connection refused (error 111)
23:33:35.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:35.778489 10479 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.37.125" port: 37371 } attempt: 3
23:33:35.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:35.782932 10479 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.37.125:37371: Network error: Client connection negotiation failed: client connection to 127.10.37.125:37371: connect: Connection refused (error 111)
23:33:35.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:35.839675 10482 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:33:35.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:35.840363 10482 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:33:35.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:35.840946 10482 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:33:35.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:35.872467 10482 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
23:33:35.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:35.872789 10482 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:33:35.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:35.872982 10482 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
23:33:35.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:35.873162 10482 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
23:33:35.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:35.873338 10482 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
23:33:35.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:35.873592 10482 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43035/jwks.json
23:33:35.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:35.909825 10482 master_runner.cc:386] Master server non-default flags:
23:33:35.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:33:35.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.37.84:38877
23:33:35.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:33:35.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:33:35.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/master-1/data
23:33:35.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/master-1/wal
23:33:35.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:33:35.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.37.126:46523,127.10.37.125:37371,127.10.37.124:38379
23:33:35.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:33:35.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:33:35.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:33:35.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.37.125:37371
23:33:35.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:33:35.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:33:35.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:33:35.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43035/jwks.json
23:33:35.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:33:35.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:33:35.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/master-1/data/info.pb
23:33:35.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.37.125
23:33:35.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:33:35.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:33:35.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:33:35.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:33:35.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:33:35.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:33:35.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/master-1/logs
23:33:35.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:33:35.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:33:35.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
23:33:35.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
23:33:35.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:33:35.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:33:35.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:33:35.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 20:03:34 UTC on ccycloud.quasar-agiwfo.root.comops.site
23:33:35.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4576543
23:33:35.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:33:35.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:35.911185 10482 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:33:35.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:35.912997 10482 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:33:35.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:35.926508 10479 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.37.125" port: 37371 } attempt: 4
23:33:35.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:35.932622 10479 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.37.125:37371: Network error: Client connection negotiation failed: client connection to 127.10.37.125:37371: connect: Connection refused (error 111)
23:33:35.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:35.935124 10492 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:33:36.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:36.232134 10479 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.37.125" port: 37371 } attempt: 5
23:33:36.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:36.237758 10479 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.37.125:37371: Network error: Client connection negotiation failed: client connection to 127.10.37.125:37371: connect: Connection refused (error 111)
23:33:36.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:36.785272 10479 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.37.125" port: 37371 } attempt: 6
23:33:36.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:36.793625 10479 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.37.125:37371: Network error: Client connection negotiation failed: client connection to 127.10.37.125:37371: connect: Connection refused (error 111)
23:33:37.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:35.935142 10493 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:33:37.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:35.937354 10495 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:33:37.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:37.131913 10494 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
23:33:37.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:37.131983 10482 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
23:33:37.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:37.137120 10482 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:33:37.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:37.140980 10482 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:33:37.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:37.142481 10482 hybrid_clock.cc:648] HybridClock initialized: now 1786318417142435 us; error 65 us; skew 500 ppm
23:33:37.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:37.143471 10482 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:33:37.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:37.155117 10482 webserver.cc:460] Webserver started at http://127.10.37.125:43661/ using document root <none> and password file <none>
23:33:37.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:37.156083 10482 fs_manager.cc:359] Metadata directory not provided
23:33:37.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:37.156293 10482 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:33:37.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:37.156699 10482 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
23:33:37.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:37.161885 10482 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/master-1/data/instance:
23:33:37.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3521c9b4fe344205992a8f93975e6de0"
23:33:37.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 23:33:37 on dist-test-slave-7c35"
23:33:37.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:37.163192 10482 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/master-1/wal/instance:
23:33:37.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3521c9b4fe344205992a8f93975e6de0"
23:33:37.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 23:33:37 on dist-test-slave-7c35"
23:33:37.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:37.170974 10482 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.007s sys 0.000s
23:33:37.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:37.177028 10505 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
23:33:37.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:37.178278 10482 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.006s sys 0.000s
23:33:37.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:37.178699 10482 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/master-1/data,/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/master-1/wal
23:33:37.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3521c9b4fe344205992a8f93975e6de0"
23:33:37.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 23:33:37 on dist-test-slave-7c35"
23:33:37.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:37.179234 10482 fs_report.cc:383] FS layout report
23:33:37.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:33:37.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/master-1/wal
23:33:37.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/master-1/wal
23:33:37.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/master-1/data/data
23:33:37.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:33:37.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:33:37.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:33:37.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:33:37.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:33:37.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:33:37.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:33:37.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:33:37.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:33:37.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:33:37.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:33:37.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:37.224985 10482 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
23:33:37.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:37.226769 10482 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:33:37.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:37.227402 10482 kserver.cc:163] Server-wide thread pool size limit: 3276
23:33:37.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:37.308099 10482 rpc_server.cc:307] RPC server started. Bound to: 127.10.37.125:37371
23:33:37.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:37.308177 10561 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.37.125:37371 every 8 connection(s)
23:33:37.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:37.311120 10482 server_base.cc:1166] Dumped server information to /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/master-1/data/info.pb
23:33:37.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:37.316620 10562 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
23:33:37.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:37.319548 10389 external_mini_cluster.cc:1388] Started /tmp/dist-test-task5zWf6n/build/tsan/bin/kudu as pid 10482
23:33:37.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:37.320010 10389 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/master-1/wal/instance
23:33:37.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:37.322229 10389 external_mini_cluster.cc:1326] Running /tmp/dist-test-task5zWf6n/build/tsan/bin/kudu
23:33:37.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task5zWf6n/build/tsan/bin/kudu
23:33:37.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/master-2/wal
23:33:37.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/master-2/data
23:33:37.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:33:37.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:33:37.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:33:37.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:33:37.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:33:37.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:33:37.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/master-2/logs
23:33:37.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/master-2/data/info.pb
23:33:37.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:33:37.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:33:37.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:33:37.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:33:37.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:33:37.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:33:37.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:33:37.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:33:37.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
23:33:37.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:33:37.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:33:37.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:33:37.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.37.124:38379
23:33:37.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.37.124
23:33:37.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:33:37.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.37.84:38877
23:33:37.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:33:37.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:33:37.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:33:37.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:33:37.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.37.126:46523,127.10.37.125:37371,127.10.37.124:38379
23:33:37.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:33:37.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43035/jwks.json with env {}
23:33:37.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:37.328959 10562 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.37.126" port: 46523 } has no permanent_uuid. Determining permanent_uuid...
23:33:37.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:37.356150 10562 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.37.125" port: 37371 } has no permanent_uuid. Determining permanent_uuid...
23:33:37.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:37.373219 10562 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.37.124" port: 38379 } has no permanent_uuid. Determining permanent_uuid...
23:33:37.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:37.376077 10509 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.37.124:38379: connect: Connection refused (error 111)
23:33:37.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:37.380389 10562 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.37.124:38379: Network error: Client connection negotiation failed: client connection to 127.10.37.124:38379: connect: Connection refused (error 111)
23:33:37.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:37.429976 10562 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.37.124" port: 38379 } attempt: 1
23:33:37.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:37.434965 10562 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.37.124:38379: Network error: Client connection negotiation failed: client connection to 127.10.37.124:38379: connect: Connection refused (error 111)
23:33:37.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:37.503458 10562 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.37.124" port: 38379 } attempt: 2
23:33:37.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:37.507548 10562 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.37.124:38379: Network error: Client connection negotiation failed: client connection to 127.10.37.124:38379: connect: Connection refused (error 111)
23:33:37.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:37.599064 10562 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.37.124" port: 38379 } attempt: 3
23:33:37.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:37.603627 10562 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.37.124:38379: Network error: Client connection negotiation failed: client connection to 127.10.37.124:38379: connect: Connection refused (error 111)
23:33:37.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:37.666437 10564 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:33:37.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:37.667122 10564 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:33:37.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:37.667636 10564 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:33:37.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:37.699753 10564 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
23:33:37.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:37.700166 10564 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:33:37.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:37.700486 10564 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
23:33:37.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:37.700713 10564 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
23:33:37.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:37.700997 10564 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
23:33:37.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:37.701252 10564 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43035/jwks.json
23:33:37.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:37.737426 10564 master_runner.cc:386] Master server non-default flags:
23:33:37.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:33:37.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.37.84:38877
23:33:37.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:33:37.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:33:37.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/master-2/data
23:33:37.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/master-2/wal
23:33:37.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:33:37.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.37.126:46523,127.10.37.125:37371,127.10.37.124:38379
23:33:37.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:33:37.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:33:37.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:33:37.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.37.124:38379
23:33:37.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:33:37.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:33:37.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:33:37.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43035/jwks.json
23:33:37.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:33:37.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:33:37.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/master-2/data/info.pb
23:33:37.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.37.124
23:33:37.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:33:37.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:33:37.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:33:37.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:33:37.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:33:37.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:33:37.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/master-2/logs
23:33:37.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:33:37.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:33:37.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
23:33:37.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
23:33:37.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:33:37.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:33:37.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:33:37.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 20:03:34 UTC on ccycloud.quasar-agiwfo.root.comops.site
23:33:37.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4576543
23:33:37.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:33:37.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:37.738730 10564 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:33:37.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:37.740859 10564 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:33:37.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:37.747116 10562 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.37.124" port: 38379 } attempt: 4
23:33:37.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:37.752640 10562 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.37.124:38379: Network error: Client connection negotiation failed: client connection to 127.10.37.124:38379: connect: Connection refused (error 111)
23:33:37.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:37.761833 10578 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:33:37.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:37.854079 10479 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.37.125" port: 37371 } attempt: 7
23:33:37.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:37.867928 10479 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.37.124" port: 38379 } has no permanent_uuid. Determining permanent_uuid...
23:33:37.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:37.873432 10479 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.37.124:38379: Network error: Client connection negotiation failed: client connection to 127.10.37.124:38379: connect: Connection refused (error 111)
23:33:37.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:37.932139 10479 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.37.124" port: 38379 } attempt: 1
23:33:37.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:37.938541 10479 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.37.124:38379: Network error: Client connection negotiation failed: client connection to 127.10.37.124:38379: connect: Connection refused (error 111)
23:33:38.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:38.019991 10479 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.37.124" port: 38379 } attempt: 2
23:33:38.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:38.025939 10479 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.37.124:38379: Network error: Client connection negotiation failed: client connection to 127.10.37.124:38379: connect: Connection refused (error 111)
23:33:38.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:38.052101 10562 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.37.124" port: 38379 } attempt: 5
23:33:38.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:38.057163 10562 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.37.124:38379: Network error: Client connection negotiation failed: client connection to 127.10.37.124:38379: connect: Connection refused (error 111)
23:33:38.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:38.111455 10479 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.37.124" port: 38379 } attempt: 3
23:33:38.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:38.116864 10479 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.37.124:38379: Network error: Client connection negotiation failed: client connection to 127.10.37.124:38379: connect: Connection refused (error 111)
23:33:38.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:38.257318 10479 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.37.124" port: 38379 } attempt: 4
23:33:38.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:38.262147 10479 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.37.124:38379: Network error: Client connection negotiation failed: client connection to 127.10.37.124:38379: connect: Connection refused (error 111)
23:33:38.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:38.545547 10479 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.37.124" port: 38379 } attempt: 5
23:33:38.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:38.550364 10479 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.37.124:38379: Network error: Client connection negotiation failed: client connection to 127.10.37.124:38379: connect: Connection refused (error 111)
23:33:38.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:38.604583 10562 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.37.124" port: 38379 } attempt: 6
23:33:38.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:38.612989 10562 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.37.124:38379: Network error: Client connection negotiation failed: client connection to 127.10.37.124:38379: connect: Connection refused (error 111)
23:33:38.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:37.762576 10575 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:33:38.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:37.764436 10564 server_base.cc:1034] running on GCE node
23:33:38.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:37.763839 10576 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:33:38.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:38.987929 10564 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:33:38.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:38.990957 10564 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:33:38.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:38.992458 10564 hybrid_clock.cc:648] HybridClock initialized: now 1786318418992382 us; error 58 us; skew 500 ppm
23:33:38.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:38.993577 10564 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:33:39.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.001464 10564 webserver.cc:460] Webserver started at http://127.10.37.124:33793/ using document root <none> and password file <none>
23:33:39.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.002821 10564 fs_manager.cc:359] Metadata directory not provided
23:33:39.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.003180 10564 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:33:39.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.003844 10564 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
23:33:39.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.009569 10564 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/master-2/data/instance:
23:33:39.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "80b9c910dfbc4cf88b8f794612e66d56"
23:33:39.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 23:33:39 on dist-test-slave-7c35"
23:33:39.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.011090 10564 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/master-2/wal/instance:
23:33:39.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "80b9c910dfbc4cf88b8f794612e66d56"
23:33:39.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 23:33:39 on dist-test-slave-7c35"
23:33:39.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.020359 10564 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.007s sys 0.001s
23:33:39.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.027258 10590 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
23:33:39.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.028503 10564 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
23:33:39.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.028934 10564 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/master-2/data,/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/master-2/wal
23:33:39.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "80b9c910dfbc4cf88b8f794612e66d56"
23:33:39.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 23:33:39 on dist-test-slave-7c35"
23:33:39.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.029450 10564 fs_report.cc:383] FS layout report
23:33:39.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:33:39.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/master-2/wal
23:33:39.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/master-2/wal
23:33:39.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/master-2/data/data
23:33:39.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:33:39.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:33:39.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:33:39.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:33:39.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:33:39.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:33:39.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:33:39.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:33:39.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:33:39.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:33:39.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:33:39.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.082718 10564 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
23:33:39.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.084694 10564 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:33:39.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.085309 10564 kserver.cc:163] Server-wide thread pool size limit: 3276
23:33:39.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.102787 10479 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.37.124" port: 38379 } attempt: 6
23:33:39.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:39.111225 10479 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.37.124:38379: Network error: Client connection negotiation failed: client connection to 127.10.37.124:38379: connect: Connection refused (error 111)
23:33:39.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.168792 10564 rpc_server.cc:307] RPC server started. Bound to: 127.10.37.124:38379
23:33:39.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.168876 10650 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.37.124:38379 every 8 connection(s)
23:33:39.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.171514 10564 server_base.cc:1166] Dumped server information to /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/master-2/data/info.pb
23:33:39.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.177533 10651 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
23:33:39.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.178994 10389 external_mini_cluster.cc:1388] Started /tmp/dist-test-task5zWf6n/build/tsan/bin/kudu as pid 10564
23:33:39.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.179344 10389 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/master-2/wal/instance
23:33:39.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.193647 10651 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.37.126" port: 46523 } has no permanent_uuid. Determining permanent_uuid...
23:33:39.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.226946 10651 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.37.125" port: 37371 } has no permanent_uuid. Determining permanent_uuid...
23:33:39.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.240494 10651 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.37.124" port: 38379 } has no permanent_uuid. Determining permanent_uuid...
23:33:39.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.261603 10651 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 80b9c910dfbc4cf88b8f794612e66d56: Bootstrap starting.
23:33:39.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.270569 10651 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 80b9c910dfbc4cf88b8f794612e66d56: Neither blocks nor log segments found. Creating new log.
23:33:39.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.274056 10651 log.cc:824] T 00000000000000000000000000000000 P 80b9c910dfbc4cf88b8f794612e66d56: Log is configured to *not* fsync() on all Append() calls
23:33:39.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.280726 10651 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 80b9c910dfbc4cf88b8f794612e66d56: No bootstrap required, opened a new log
23:33:39.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.307201 10651 raft_consensus.cc:348] T 00000000000000000000000000000000 P 80b9c910dfbc4cf88b8f794612e66d56 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c6ad93e58f848efa29d1594b7f18918" member_type: VOTER last_known_addr { host: "127.10.37.126" port: 46523 } } peers { permanent_uuid: "3521c9b4fe344205992a8f93975e6de0" member_type: VOTER last_known_addr { host: "127.10.37.125" port: 37371 } } peers { permanent_uuid: "80b9c910dfbc4cf88b8f794612e66d56" member_type: VOTER last_known_addr { host: "127.10.37.124" port: 38379 } }
23:33:39.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.308725 10651 raft_consensus.cc:374] T 00000000000000000000000000000000 P 80b9c910dfbc4cf88b8f794612e66d56 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
23:33:39.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.309499 10651 raft_consensus.cc:729] T 00000000000000000000000000000000 P 80b9c910dfbc4cf88b8f794612e66d56 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 80b9c910dfbc4cf88b8f794612e66d56, State: Initialized, Role: FOLLOWER
23:33:39.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.311250 10651 consensus_queue.cc:260] T 00000000000000000000000000000000 P 80b9c910dfbc4cf88b8f794612e66d56 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c6ad93e58f848efa29d1594b7f18918" member_type: VOTER last_known_addr { host: "127.10.37.126" port: 46523 } } peers { permanent_uuid: "3521c9b4fe344205992a8f93975e6de0" member_type: VOTER last_known_addr { host: "127.10.37.125" port: 37371 } } peers { permanent_uuid: "80b9c910dfbc4cf88b8f794612e66d56" member_type: VOTER last_known_addr { host: "127.10.37.124" port: 38379 } }
23:33:39.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.314428 10661 sys_catalog.cc:455] T 00000000000000000000000000000000 P 80b9c910dfbc4cf88b8f794612e66d56 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c6ad93e58f848efa29d1594b7f18918" member_type: VOTER last_known_addr { host: "127.10.37.126" port: 46523 } } peers { permanent_uuid: "3521c9b4fe344205992a8f93975e6de0" member_type: VOTER last_known_addr { host: "127.10.37.125" port: 37371 } } peers { permanent_uuid: "80b9c910dfbc4cf88b8f794612e66d56" member_type: VOTER last_known_addr { host: "127.10.37.124" port: 38379 } } }
23:33:39.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.315413 10661 sys_catalog.cc:458] T 00000000000000000000000000000000 P 80b9c910dfbc4cf88b8f794612e66d56 [sys.catalog]: This master's current role is: FOLLOWER
23:33:39.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.316244 10651 sys_catalog.cc:564] T 00000000000000000000000000000000 P 80b9c910dfbc4cf88b8f794612e66d56 [sys.catalog]: configured and running, proceeding with master startup.
23:33:39.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:39.342320 10597 tablet.cc:2369] T 00000000000000000000000000000000 P 80b9c910dfbc4cf88b8f794612e66d56: Can't schedule compaction. Clean time has not been advanced past its initial value.
23:33:39.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:39.349725 10673 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 80b9c910dfbc4cf88b8f794612e66d56: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:33:39.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:39.350030 10673 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:33:39.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.346040 10564 master_runner.cc:186] Error getting master registration for 127.10.37.126:46523: OK, instance_id { permanent_uuid: "4c6ad93e58f848efa29d1594b7f18918" instance_seqno: 1786318415394360 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
23:33:39.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.374625 10564 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
23:33:39.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.397020 10661 raft_consensus.cc:482] T 00000000000000000000000000000000 P 80b9c910dfbc4cf88b8f794612e66d56 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
23:33:39.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.397697 10661 raft_consensus.cc:504] T 00000000000000000000000000000000 P 80b9c910dfbc4cf88b8f794612e66d56 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c6ad93e58f848efa29d1594b7f18918" member_type: VOTER last_known_addr { host: "127.10.37.126" port: 46523 } } peers { permanent_uuid: "3521c9b4fe344205992a8f93975e6de0" member_type: VOTER last_known_addr { host: "127.10.37.125" port: 37371 } } peers { permanent_uuid: "80b9c910dfbc4cf88b8f794612e66d56" member_type: VOTER last_known_addr { host: "127.10.37.124" port: 38379 } }
23:33:39.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.402129 10661 leader_election.cc:288] T 00000000000000000000000000000000 P 80b9c910dfbc4cf88b8f794612e66d56 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 4c6ad93e58f848efa29d1594b7f18918 (127.10.37.126:46523), 3521c9b4fe344205992a8f93975e6de0 (127.10.37.125:37371)
23:33:39.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.673425 10562 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.37.124" port: 38379 } attempt: 7
23:33:39.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.699182 10562 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3521c9b4fe344205992a8f93975e6de0: Bootstrap starting.
23:33:39.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.707403 10562 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 3521c9b4fe344205992a8f93975e6de0: Neither blocks nor log segments found. Creating new log.
23:33:39.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.709972 10562 log.cc:824] T 00000000000000000000000000000000 P 3521c9b4fe344205992a8f93975e6de0: Log is configured to *not* fsync() on all Append() calls
23:33:39.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.715778 10562 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3521c9b4fe344205992a8f93975e6de0: No bootstrap required, opened a new log
23:33:39.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.750558 10562 raft_consensus.cc:348] T 00000000000000000000000000000000 P 3521c9b4fe344205992a8f93975e6de0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c6ad93e58f848efa29d1594b7f18918" member_type: VOTER last_known_addr { host: "127.10.37.126" port: 46523 } } peers { permanent_uuid: "3521c9b4fe344205992a8f93975e6de0" member_type: VOTER last_known_addr { host: "127.10.37.125" port: 37371 } } peers { permanent_uuid: "80b9c910dfbc4cf88b8f794612e66d56" member_type: VOTER last_known_addr { host: "127.10.37.124" port: 38379 } }
23:33:39.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.751881 10562 raft_consensus.cc:374] T 00000000000000000000000000000000 P 3521c9b4fe344205992a8f93975e6de0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
23:33:39.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.752629 10562 raft_consensus.cc:729] T 00000000000000000000000000000000 P 3521c9b4fe344205992a8f93975e6de0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3521c9b4fe344205992a8f93975e6de0, State: Initialized, Role: FOLLOWER
23:33:39.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.753746 10562 consensus_queue.cc:260] T 00000000000000000000000000000000 P 3521c9b4fe344205992a8f93975e6de0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c6ad93e58f848efa29d1594b7f18918" member_type: VOTER last_known_addr { host: "127.10.37.126" port: 46523 } } peers { permanent_uuid: "3521c9b4fe344205992a8f93975e6de0" member_type: VOTER last_known_addr { host: "127.10.37.125" port: 37371 } } peers { permanent_uuid: "80b9c910dfbc4cf88b8f794612e66d56" member_type: VOTER last_known_addr { host: "127.10.37.124" port: 38379 } }
23:33:39.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.757057 10678 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3521c9b4fe344205992a8f93975e6de0 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c6ad93e58f848efa29d1594b7f18918" member_type: VOTER last_known_addr { host: "127.10.37.126" port: 46523 } } peers { permanent_uuid: "3521c9b4fe344205992a8f93975e6de0" member_type: VOTER last_known_addr { host: "127.10.37.125" port: 37371 } } peers { permanent_uuid: "80b9c910dfbc4cf88b8f794612e66d56" member_type: VOTER last_known_addr { host: "127.10.37.124" port: 38379 } } }
23:33:39.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.757917 10678 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3521c9b4fe344205992a8f93975e6de0 [sys.catalog]: This master's current role is: FOLLOWER
23:33:39.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.759009 10562 sys_catalog.cc:564] T 00000000000000000000000000000000 P 3521c9b4fe344205992a8f93975e6de0 [sys.catalog]: configured and running, proceeding with master startup.
23:33:39.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.759433 10534 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "80b9c910dfbc4cf88b8f794612e66d56" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3521c9b4fe344205992a8f93975e6de0" is_pre_election: true
23:33:39.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.760511 10534 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 3521c9b4fe344205992a8f93975e6de0 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 80b9c910dfbc4cf88b8f794612e66d56 in term 0.
23:33:39.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.762074 10593 leader_election.cc:302] T 00000000000000000000000000000000 P 80b9c910dfbc4cf88b8f794612e66d56 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3521c9b4fe344205992a8f93975e6de0, 80b9c910dfbc4cf88b8f794612e66d56; no voters:
23:33:39.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.763192 10661 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 80b9c910dfbc4cf88b8f794612e66d56 [term 0 FOLLOWER]: Leader pre-election won for term 1
23:33:39.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.763527 10661 raft_consensus.cc:482] T 00000000000000000000000000000000 P 80b9c910dfbc4cf88b8f794612e66d56 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
23:33:39.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.763842 10661 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 80b9c910dfbc4cf88b8f794612e66d56 [term 0 FOLLOWER]: Advancing to term 1
23:33:39.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.769611 10661 raft_consensus.cc:504] T 00000000000000000000000000000000 P 80b9c910dfbc4cf88b8f794612e66d56 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c6ad93e58f848efa29d1594b7f18918" member_type: VOTER last_known_addr { host: "127.10.37.126" port: 46523 } } peers { permanent_uuid: "3521c9b4fe344205992a8f93975e6de0" member_type: VOTER last_known_addr { host: "127.10.37.125" port: 37371 } } peers { permanent_uuid: "80b9c910dfbc4cf88b8f794612e66d56" member_type: VOTER last_known_addr { host: "127.10.37.124" port: 38379 } }
23:33:39.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.772344 10661 leader_election.cc:288] T 00000000000000000000000000000000 P 80b9c910dfbc4cf88b8f794612e66d56 [CANDIDATE]: Term 1 election: Requested vote from peers 4c6ad93e58f848efa29d1594b7f18918 (127.10.37.126:46523), 3521c9b4fe344205992a8f93975e6de0 (127.10.37.125:37371)
23:33:39.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.774704 10534 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "80b9c910dfbc4cf88b8f794612e66d56" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3521c9b4fe344205992a8f93975e6de0"
23:33:39.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.775388 10534 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 3521c9b4fe344205992a8f93975e6de0 [term 0 FOLLOWER]: Advancing to term 1
23:33:39.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.782346 10534 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 3521c9b4fe344205992a8f93975e6de0 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 80b9c910dfbc4cf88b8f794612e66d56 in term 1.
23:33:39.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.783793 10593 leader_election.cc:302] T 00000000000000000000000000000000 P 80b9c910dfbc4cf88b8f794612e66d56 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3521c9b4fe344205992a8f93975e6de0, 80b9c910dfbc4cf88b8f794612e66d56; no voters:
23:33:39.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.784647 10661 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 80b9c910dfbc4cf88b8f794612e66d56 [term 1 FOLLOWER]: Leader election won for term 1
23:33:39.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.787871 10661 raft_consensus.cc:686] T 00000000000000000000000000000000 P 80b9c910dfbc4cf88b8f794612e66d56 [term 1 LEADER]: Becoming Leader. State: Replica: 80b9c910dfbc4cf88b8f794612e66d56, State: Running, Role: LEADER
23:33:39.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.788919 10661 consensus_queue.cc:237] T 00000000000000000000000000000000 P 80b9c910dfbc4cf88b8f794612e66d56 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c6ad93e58f848efa29d1594b7f18918" member_type: VOTER last_known_addr { host: "127.10.37.126" port: 46523 } } peers { permanent_uuid: "3521c9b4fe344205992a8f93975e6de0" member_type: VOTER last_known_addr { host: "127.10.37.125" port: 37371 } } peers { permanent_uuid: "80b9c910dfbc4cf88b8f794612e66d56" member_type: VOTER last_known_addr { host: "127.10.37.124" port: 38379 } }
23:33:39.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.815758 10683 sys_catalog.cc:455] T 00000000000000000000000000000000 P 80b9c910dfbc4cf88b8f794612e66d56 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 80b9c910dfbc4cf88b8f794612e66d56. Latest consensus state: current_term: 1 leader_uuid: "80b9c910dfbc4cf88b8f794612e66d56" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c6ad93e58f848efa29d1594b7f18918" member_type: VOTER last_known_addr { host: "127.10.37.126" port: 46523 } } peers { permanent_uuid: "3521c9b4fe344205992a8f93975e6de0" member_type: VOTER last_known_addr { host: "127.10.37.125" port: 37371 } } peers { permanent_uuid: "80b9c910dfbc4cf88b8f794612e66d56" member_type: VOTER last_known_addr { host: "127.10.37.124" port: 38379 } } }
23:33:39.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.816618 10683 sys_catalog.cc:458] T 00000000000000000000000000000000 P 80b9c910dfbc4cf88b8f794612e66d56 [sys.catalog]: This master's current role is: LEADER
23:33:39.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.820461 10689 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
23:33:39.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.829713 10689 catalog_manager.cc:1484] Initializing Kudu cluster ID...
23:33:39.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:39.875874 10692 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3521c9b4fe344205992a8f93975e6de0: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:33:39.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:39.877782 10692 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:33:39.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.851881 10482 master_runner.cc:186] Error getting master registration for 127.10.37.126:46523: OK, instance_id { permanent_uuid: "4c6ad93e58f848efa29d1594b7f18918" instance_seqno: 1786318415394360 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
23:33:39.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.889065 10482 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
23:33:39.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.892154 10534 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 3521c9b4fe344205992a8f93975e6de0 [term 1 FOLLOWER]: Refusing update from remote peer 80b9c910dfbc4cf88b8f794612e66d56: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
23:33:39.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.894263 10683 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 80b9c910dfbc4cf88b8f794612e66d56 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3521c9b4fe344205992a8f93975e6de0" member_type: VOTER last_known_addr { host: "127.10.37.125" port: 37371 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
23:33:39.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.926578 10678 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3521c9b4fe344205992a8f93975e6de0 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 80b9c910dfbc4cf88b8f794612e66d56. Latest consensus state: current_term: 1 leader_uuid: "80b9c910dfbc4cf88b8f794612e66d56" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c6ad93e58f848efa29d1594b7f18918" member_type: VOTER last_known_addr { host: "127.10.37.126" port: 46523 } } peers { permanent_uuid: "3521c9b4fe344205992a8f93975e6de0" member_type: VOTER last_known_addr { host: "127.10.37.125" port: 37371 } } peers { permanent_uuid: "80b9c910dfbc4cf88b8f794612e66d56" member_type: VOTER last_known_addr { host: "127.10.37.124" port: 38379 } } }
23:33:39.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.929677 10678 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3521c9b4fe344205992a8f93975e6de0 [sys.catalog]: This master's current role is: FOLLOWER
23:33:39.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.933990 10694 mvcc.cc:205] Tried to move back new op lower bound from 7316760247786049536 to 7316760247461945344. Current Snapshot: MvccSnapshot[applied={T|T < 7316760247786049536}]
23:33:39.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.940569 10696 mvcc.cc:205] Tried to move back new op lower bound from 7316760247786049536 to 7316760247461945344. Current Snapshot: MvccSnapshot[applied={T|T < 7316760247786049536}]
23:33:39.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.942389 10661 sys_catalog.cc:455] T 00000000000000000000000000000000 P 80b9c910dfbc4cf88b8f794612e66d56 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "80b9c910dfbc4cf88b8f794612e66d56" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c6ad93e58f848efa29d1594b7f18918" member_type: VOTER last_known_addr { host: "127.10.37.126" port: 46523 } } peers { permanent_uuid: "3521c9b4fe344205992a8f93975e6de0" member_type: VOTER last_known_addr { host: "127.10.37.125" port: 37371 } } peers { permanent_uuid: "80b9c910dfbc4cf88b8f794612e66d56" member_type: VOTER last_known_addr { host: "127.10.37.124" port: 38379 } } }
23:33:39.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.943280 10661 sys_catalog.cc:458] T 00000000000000000000000000000000 P 80b9c910dfbc4cf88b8f794612e66d56 [sys.catalog]: This master's current role is: LEADER
23:33:39.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.947314 10689 catalog_manager.cc:1347] Generated new cluster ID: e839910c729347d1982a2967ae4203ed
23:33:39.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.947631 10689 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
23:33:39.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.955637 10678 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3521c9b4fe344205992a8f93975e6de0 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "80b9c910dfbc4cf88b8f794612e66d56" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c6ad93e58f848efa29d1594b7f18918" member_type: VOTER last_known_addr { host: "127.10.37.126" port: 46523 } } peers { permanent_uuid: "3521c9b4fe344205992a8f93975e6de0" member_type: VOTER last_known_addr { host: "127.10.37.125" port: 37371 } } peers { permanent_uuid: "80b9c910dfbc4cf88b8f794612e66d56" member_type: VOTER last_known_addr { host: "127.10.37.124" port: 38379 } } }
23:33:39.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:39.956404 10678 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3521c9b4fe344205992a8f93975e6de0 [sys.catalog]: This master's current role is: FOLLOWER
23:33:40.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:40.023433 10689 catalog_manager.cc:1370] Generated new certificate authority record
23:33:40.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:40.030676 10689 catalog_manager.cc:1504] Loading token signing keys...
23:33:40.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:40.079572 10689 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 80b9c910dfbc4cf88b8f794612e66d56: Generated new TSK 0
23:33:40.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:40.082417 10689 catalog_manager.cc:1514] Initializing in-progress tserver states...
23:33:40.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:40.127954 10389 external_mini_cluster.cc:1326] Running /tmp/dist-test-task5zWf6n/build/tsan/bin/kudu
23:33:40.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task5zWf6n/build/tsan/bin/kudu
23:33:40.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/ts-0/wal
23:33:40.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/ts-0/data
23:33:40.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:33:40.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:33:40.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:33:40.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:33:40.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:33:40.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:33:40.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/ts-0/logs
23:33:40.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/ts-0/data/info.pb
23:33:40.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:33:40.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:33:40.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:33:40.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:33:40.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:33:40.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:33:40.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:33:40.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:33:40.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:33:40.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:33:40.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.37.65:0
23:33:40.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.37.65
23:33:40.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.37.65
23:33:40.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:33:40.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.37.126:46523,127.10.37.125:37371,127.10.37.124:38379
23:33:40.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.37.84:38877
23:33:40.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:33:40.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:33:40.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:33:40.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:40.144748 10479 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.37.124" port: 38379 } attempt: 7
23:33:40.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:40.192117 10479 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4c6ad93e58f848efa29d1594b7f18918: Bootstrap starting.
23:33:40.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:40.214169 10479 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 4c6ad93e58f848efa29d1594b7f18918: Neither blocks nor log segments found. Creating new log.
23:33:40.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:40.221457 10479 log.cc:824] T 00000000000000000000000000000000 P 4c6ad93e58f848efa29d1594b7f18918: Log is configured to *not* fsync() on all Append() calls
23:33:40.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:40.264524 10479 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4c6ad93e58f848efa29d1594b7f18918: No bootstrap required, opened a new log
23:33:40.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:40.361315 10479 raft_consensus.cc:348] T 00000000000000000000000000000000 P 4c6ad93e58f848efa29d1594b7f18918 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c6ad93e58f848efa29d1594b7f18918" member_type: VOTER last_known_addr { host: "127.10.37.126" port: 46523 } } peers { permanent_uuid: "3521c9b4fe344205992a8f93975e6de0" member_type: VOTER last_known_addr { host: "127.10.37.125" port: 37371 } } peers { permanent_uuid: "80b9c910dfbc4cf88b8f794612e66d56" member_type: VOTER last_known_addr { host: "127.10.37.124" port: 38379 } }
23:33:40.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:40.365898 10479 raft_consensus.cc:374] T 00000000000000000000000000000000 P 4c6ad93e58f848efa29d1594b7f18918 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
23:33:40.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:40.367323 10479 raft_consensus.cc:729] T 00000000000000000000000000000000 P 4c6ad93e58f848efa29d1594b7f18918 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4c6ad93e58f848efa29d1594b7f18918, State: Initialized, Role: FOLLOWER
23:33:40.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:40.370018 10479 consensus_queue.cc:260] T 00000000000000000000000000000000 P 4c6ad93e58f848efa29d1594b7f18918 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c6ad93e58f848efa29d1594b7f18918" member_type: VOTER last_known_addr { host: "127.10.37.126" port: 46523 } } peers { permanent_uuid: "3521c9b4fe344205992a8f93975e6de0" member_type: VOTER last_known_addr { host: "127.10.37.125" port: 37371 } } peers { permanent_uuid: "80b9c910dfbc4cf88b8f794612e66d56" member_type: VOTER last_known_addr { host: "127.10.37.124" port: 38379 } }
23:33:40.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:40.404066 10703 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4c6ad93e58f848efa29d1594b7f18918 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c6ad93e58f848efa29d1594b7f18918" member_type: VOTER last_known_addr { host: "127.10.37.126" port: 46523 } } peers { permanent_uuid: "3521c9b4fe344205992a8f93975e6de0" member_type: VOTER last_known_addr { host: "127.10.37.125" port: 37371 } } peers { permanent_uuid: "80b9c910dfbc4cf88b8f794612e66d56" member_type: VOTER last_known_addr { host: "127.10.37.124" port: 38379 } } }
23:33:40.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:40.405012 10703 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4c6ad93e58f848efa29d1594b7f18918 [sys.catalog]: This master's current role is: FOLLOWER
23:33:40.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:40.412192 10449 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 4c6ad93e58f848efa29d1594b7f18918 [term 0 FOLLOWER]: Advancing to term 1
23:33:40.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:40.415120 10451 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "80b9c910dfbc4cf88b8f794612e66d56" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4c6ad93e58f848efa29d1594b7f18918" is_pre_election: true
23:33:40.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:40.416290 10451 rpcz_store.cc:275] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:46376 (request call id 2) took 1014 ms. Trace:
23:33:40.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:40.416608 10451 rpcz_store.cc:276] 0809 23:33:39.402323 (+ 0us) service_pool.cc:170] Inserting onto call queue
23:33:40.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0809 23:33:39.402505 (+ 182us) service_pool.cc:227] Handling call
23:33:40.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0809 23:33:40.416224 (+1013719us) inbound_call.cc:173] Queueing success response
23:33:40.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
23:33:40.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:40.417829 10450 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "80b9c910dfbc4cf88b8f794612e66d56" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4c6ad93e58f848efa29d1594b7f18918"
23:33:40.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:40.426235 10449 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 4c6ad93e58f848efa29d1594b7f18918 [term 1 FOLLOWER]: Refusing update from remote peer 80b9c910dfbc4cf88b8f794612e66d56: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
23:33:40.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:40.428534 10683 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 80b9c910dfbc4cf88b8f794612e66d56 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c6ad93e58f848efa29d1594b7f18918" member_type: VOTER last_known_addr { host: "127.10.37.126" port: 46523 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
23:33:40.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:40.439694 10479 sys_catalog.cc:564] T 00000000000000000000000000000000 P 4c6ad93e58f848efa29d1594b7f18918 [sys.catalog]: configured and running, proceeding with master startup.
23:33:40.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:40.505998 10703 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4c6ad93e58f848efa29d1594b7f18918 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 80b9c910dfbc4cf88b8f794612e66d56. Latest consensus state: current_term: 1 leader_uuid: "80b9c910dfbc4cf88b8f794612e66d56" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c6ad93e58f848efa29d1594b7f18918" member_type: VOTER last_known_addr { host: "127.10.37.126" port: 46523 } } peers { permanent_uuid: "3521c9b4fe344205992a8f93975e6de0" member_type: VOTER last_known_addr { host: "127.10.37.125" port: 37371 } } peers { permanent_uuid: "80b9c910dfbc4cf88b8f794612e66d56" member_type: VOTER last_known_addr { host: "127.10.37.124" port: 38379 } } }
23:33:40.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:40.506909 10703 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4c6ad93e58f848efa29d1594b7f18918 [sys.catalog]: This master's current role is: FOLLOWER
23:33:40.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:40.506186 10709 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4c6ad93e58f848efa29d1594b7f18918 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "80b9c910dfbc4cf88b8f794612e66d56" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c6ad93e58f848efa29d1594b7f18918" member_type: VOTER last_known_addr { host: "127.10.37.126" port: 46523 } } peers { permanent_uuid: "3521c9b4fe344205992a8f93975e6de0" member_type: VOTER last_known_addr { host: "127.10.37.125" port: 37371 } } peers { permanent_uuid: "80b9c910dfbc4cf88b8f794612e66d56" member_type: VOTER last_known_addr { host: "127.10.37.124" port: 38379 } } }
23:33:40.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:40.508373 10709 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4c6ad93e58f848efa29d1594b7f18918 [sys.catalog]: This master's current role is: FOLLOWER
23:33:40.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:40.586472 10719 catalog_manager.cc:1259] Loaded cluster ID: e839910c729347d1982a2967ae4203ed
23:33:40.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:40.586746 10719 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 4c6ad93e58f848efa29d1594b7f18918: loading cluster ID for follower catalog manager: success
23:33:40.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:40.599390 10719 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 4c6ad93e58f848efa29d1594b7f18918: acquiring CA information for follower catalog manager: success
23:33:40.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:40.618690 10719 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 4c6ad93e58f848efa29d1594b7f18918: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
23:33:40.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:40.832032 10700 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:33:40.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:40.833170 10700 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:33:40.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:40.834321 10700 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:33:40.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:40.883970 10692 catalog_manager.cc:1259] Loaded cluster ID: e839910c729347d1982a2967ae4203ed
23:33:40.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:40.884266 10692 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 3521c9b4fe344205992a8f93975e6de0: loading cluster ID for follower catalog manager: success
23:33:40.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:40.900782 10700 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:33:40.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:40.904378 10700 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.37.65
23:33:40.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:40.904356 10692 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 3521c9b4fe344205992a8f93975e6de0: acquiring CA information for follower catalog manager: success
23:33:40.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:40.906144 10683 sys_catalog.cc:455] T 00000000000000000000000000000000 P 80b9c910dfbc4cf88b8f794612e66d56 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "80b9c910dfbc4cf88b8f794612e66d56" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4c6ad93e58f848efa29d1594b7f18918" member_type: VOTER last_known_addr { host: "127.10.37.126" port: 46523 } } peers { permanent_uuid: "3521c9b4fe344205992a8f93975e6de0" member_type: VOTER last_known_addr { host: "127.10.37.125" port: 37371 } } peers { permanent_uuid: "80b9c910dfbc4cf88b8f794612e66d56" member_type: VOTER last_known_addr { host: "127.10.37.124" port: 38379 } } }
23:33:40.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:40.906931 10683 sys_catalog.cc:458] T 00000000000000000000000000000000 P 80b9c910dfbc4cf88b8f794612e66d56 [sys.catalog]: This master's current role is: LEADER
23:33:40.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:40.910557 10692 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 3521c9b4fe344205992a8f93975e6de0: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
23:33:40.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:40.975296 10700 tablet_server_runner.cc:78] Tablet server non-default flags:
23:33:40.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:33:40.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.37.84:38877
23:33:40.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:33:40.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:33:40.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/ts-0/data
23:33:40.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/ts-0/wal
23:33:40.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:33:40.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:33:40.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.37.65:0
23:33:40.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:33:40.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:33:40.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:33:40.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/ts-0/data/info.pb
23:33:40.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.37.65
23:33:40.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:33:40.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.37.126:46523,127.10.37.125:37371,127.10.37.124:38379
23:33:40.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:33:40.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:33:40.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:33:40.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:33:40.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:33:40.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.37.65
23:33:40.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/ts-0/logs
23:33:40.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:33:40.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:33:40.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
23:33:40.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:33:40.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:33:40.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:33:40.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:33:40.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 20:03:34 UTC on ccycloud.quasar-agiwfo.root.comops.site
23:33:40.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4576543
23:33:40.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:33:40.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:40.986114 10700 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:33:40.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:40.989073 10700 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:33:41.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:41.039280 10729 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:33:42.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:41.052373 10700 server_base.cc:1034] running on GCE node
23:33:42.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:41.053031 10728 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:33:42.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:41.044004 10732 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:33:42.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:42.599681 10700 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:33:42.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:42.603103 10700 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:33:42.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:42.604913 10700 hybrid_clock.cc:648] HybridClock initialized: now 1786318422604874 us; error 59 us; skew 500 ppm
23:33:42.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:42.605937 10700 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:33:42.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:42.619531 10700 webserver.cc:460] Webserver started at http://127.10.37.65:36571/ using document root <none> and password file <none>
23:33:42.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:42.620842 10700 fs_manager.cc:359] Metadata directory not provided
23:33:42.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:42.621176 10700 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:33:42.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:42.621790 10700 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
23:33:42.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:42.629364 10700 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/ts-0/data/instance:
23:33:42.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a1df462bce8340eb8671fea2bddf9308"
23:33:42.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 23:33:42 on dist-test-slave-7c35"
23:33:42.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:42.631062 10700 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/ts-0/wal/instance:
23:33:42.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a1df462bce8340eb8671fea2bddf9308"
23:33:42.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 23:33:42 on dist-test-slave-7c35"
23:33:42.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:42.641096 10700 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.005s sys 0.004s
23:33:42.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:42.647912 10738 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
23:33:42.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:42.649230 10700 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.002s sys 0.003s
23:33:42.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:42.649650 10700 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/ts-0/data,/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/ts-0/wal
23:33:42.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a1df462bce8340eb8671fea2bddf9308"
23:33:42.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 23:33:42 on dist-test-slave-7c35"
23:33:42.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:42.650059 10700 fs_report.cc:383] FS layout report
23:33:42.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:33:42.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/ts-0/wal
23:33:42.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/ts-0/wal
23:33:42.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/ts-0/data/data
23:33:42.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:33:42.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:33:42.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:33:42.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:33:42.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:33:42.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:33:42.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:33:42.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:33:42.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:33:42.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:33:42.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:33:42.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:42.697654 10700 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
23:33:42.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:42.699383 10700 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:33:42.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:42.699950 10700 kserver.cc:163] Server-wide thread pool size limit: 3276
23:33:42.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:42.703831 10700 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
23:33:42.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:42.709555 10700 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
23:33:42.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:42.709836 10700 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
23:33:42.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:42.710208 10700 ts_tablet_manager.cc:610] Registered 0 tablets
23:33:42.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:42.710407 10700 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
23:33:42.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:42.968950 10700 rpc_server.cc:307] RPC server started. Bound to: 127.10.37.65:43627
23:33:42.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:42.969059 10863 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.37.65:43627 every 8 connection(s)
23:33:42.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:42.972594 10700 server_base.cc:1166] Dumped server information to /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/ts-0/data/info.pb
23:33:42.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:42.978655 10389 external_mini_cluster.cc:1388] Started /tmp/dist-test-task5zWf6n/build/tsan/bin/kudu as pid 10700
23:33:42.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:42.979281 10389 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/ts-0/wal/instance
23:33:43.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:43.023775 10389 external_mini_cluster.cc:1326] Running /tmp/dist-test-task5zWf6n/build/tsan/bin/kudu
23:33:43.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task5zWf6n/build/tsan/bin/kudu
23:33:43.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/ts-1/wal
23:33:43.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/ts-1/data
23:33:43.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:33:43.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:33:43.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:33:43.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:33:43.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:33:43.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:33:43.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/ts-1/logs
23:33:43.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/ts-1/data/info.pb
23:33:43.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:33:43.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:33:43.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:33:43.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:33:43.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:33:43.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:33:43.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:33:43.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:33:43.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:33:43.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:33:43.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.37.66:0
23:33:43.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.37.66
23:33:43.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.37.66
23:33:43.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:33:43.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.37.126:46523,127.10.37.125:37371,127.10.37.124:38379
23:33:43.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.37.84:38877
23:33:43.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:33:43.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:33:43.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:33:43.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:43.056191 10866 heartbeater.cc:347] Connected to a master server at 127.10.37.126:46523
23:33:43.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:43.056771 10866 heartbeater.cc:464] Registering TS with master...
23:33:43.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:43.058322 10866 heartbeater.cc:511] Master 127.10.37.126:46523 requested a full tablet report, sending...
23:33:43.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:43.063728 10441 ts_manager.cc:194] Registered new tserver with Master: a1df462bce8340eb8671fea2bddf9308 (127.10.37.65:43627)
23:33:43.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:43.067857 10865 heartbeater.cc:347] Connected to a master server at 127.10.37.125:37371
23:33:43.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:43.068264 10865 heartbeater.cc:464] Registering TS with master...
23:33:43.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:43.069244 10865 heartbeater.cc:511] Master 127.10.37.125:37371 requested a full tablet report, sending...
23:33:43.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:43.070099 10864 heartbeater.cc:347] Connected to a master server at 127.10.37.124:38379
23:33:43.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:43.070484 10864 heartbeater.cc:464] Registering TS with master...
23:33:43.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:43.071373 10864 heartbeater.cc:511] Master 127.10.37.124:38379 requested a full tablet report, sending...
23:33:43.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:43.073537 10524 ts_manager.cc:194] Registered new tserver with Master: a1df462bce8340eb8671fea2bddf9308 (127.10.37.65:43627)
23:33:43.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:43.076409 10610 ts_manager.cc:194] Registered new tserver with Master: a1df462bce8340eb8671fea2bddf9308 (127.10.37.65:43627)
23:33:43.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:43.079857 10610 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.37.65:49569
23:33:43.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:43.514559 10874 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:33:43.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:43.515157 10874 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:33:43.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:43.515625 10874 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:33:43.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:43.549096 10874 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:33:43.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:43.549968 10874 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.37.66
23:33:43.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:43.587682 10874 tablet_server_runner.cc:78] Tablet server non-default flags:
23:33:43.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:33:43.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.37.84:38877
23:33:43.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:33:43.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:33:43.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/ts-1/data
23:33:43.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/ts-1/wal
23:33:43.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:33:43.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:33:43.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.37.66:0
23:33:43.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:33:43.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:33:43.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:33:43.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/ts-1/data/info.pb
23:33:43.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.37.66
23:33:43.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:33:43.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.37.126:46523,127.10.37.125:37371,127.10.37.124:38379
23:33:43.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:33:43.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:33:43.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:33:43.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:33:43.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:33:43.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.37.66
23:33:43.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/ts-1/logs
23:33:43.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:33:43.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:33:43.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
23:33:43.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:33:43.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:33:43.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:33:43.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:33:43.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 20:03:34 UTC on ccycloud.quasar-agiwfo.root.comops.site
23:33:43.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4576543
23:33:43.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:33:43.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:43.588915 10874 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:33:43.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:43.591029 10874 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:33:43.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:43.615072 10881 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:33:44.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:44.084789 10864 heartbeater.cc:503] Master 127.10.37.124:38379 was elected leader, sending a full tablet report...
23:33:44.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:43.616351 10884 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:33:44.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:43.622721 10882 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:33:44.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:43.617693 10874 server_base.cc:1034] running on GCE node
23:33:44.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:44.835444 10874 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:33:44.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:44.838514 10874 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:33:44.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:44.840015 10874 hybrid_clock.cc:648] HybridClock initialized: now 1786318424839965 us; error 72 us; skew 500 ppm
23:33:44.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:44.840833 10874 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:33:44.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:44.847569 10874 webserver.cc:460] Webserver started at http://127.10.37.66:42827/ using document root <none> and password file <none>
23:33:44.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:44.848528 10874 fs_manager.cc:359] Metadata directory not provided
23:33:44.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:44.848719 10874 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:33:44.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:44.849148 10874 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
23:33:44.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:44.854388 10874 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/ts-1/data/instance:
23:33:44.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "47e975160813450f9addeef8b942eb13"
23:33:44.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 23:33:44 on dist-test-slave-7c35"
23:33:44.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:44.855546 10874 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/ts-1/wal/instance:
23:33:44.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "47e975160813450f9addeef8b942eb13"
23:33:44.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 23:33:44 on dist-test-slave-7c35"
23:33:44.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:44.863284 10874 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
23:33:44.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:44.869256 10892 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
23:33:44.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:44.870455 10874 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.001s
23:33:44.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:44.870838 10874 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/ts-1/data,/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/ts-1/wal
23:33:44.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "47e975160813450f9addeef8b942eb13"
23:33:44.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 23:33:44 on dist-test-slave-7c35"
23:33:44.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:44.871285 10874 fs_report.cc:383] FS layout report
23:33:44.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:33:44.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/ts-1/wal
23:33:44.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/ts-1/wal
23:33:44.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/ts-1/data/data
23:33:44.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:33:44.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:33:44.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:33:44.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:33:44.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:33:44.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:33:44.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:33:44.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:33:44.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:33:44.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:33:44.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:33:44.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:44.912321 10874 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
23:33:44.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:44.914014 10874 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:33:44.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:44.914633 10874 kserver.cc:163] Server-wide thread pool size limit: 3276
23:33:44.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:44.917627 10874 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
23:33:44.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:44.922365 10874 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
23:33:44.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:44.922665 10874 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
23:33:44.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:44.923105 10874 ts_tablet_manager.cc:610] Registered 0 tablets
23:33:44.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:44.923317 10874 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
23:33:45.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:45.105603 10874 rpc_server.cc:307] RPC server started. Bound to: 127.10.37.66:33353
23:33:45.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:45.105719 11017 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.37.66:33353 every 8 connection(s)
23:33:45.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:45.109169 10874 server_base.cc:1166] Dumped server information to /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/ts-1/data/info.pb
23:33:45.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:45.111874 10389 external_mini_cluster.cc:1388] Started /tmp/dist-test-task5zWf6n/build/tsan/bin/kudu as pid 10874
23:33:45.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:45.112658 10389 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/ts-1/wal/instance
23:33:45.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:45.127038 10389 external_mini_cluster.cc:1326] Running /tmp/dist-test-task5zWf6n/build/tsan/bin/kudu
23:33:45.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task5zWf6n/build/tsan/bin/kudu
23:33:45.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/ts-2/wal
23:33:45.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/ts-2/data
23:33:45.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:33:45.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:33:45.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:33:45.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:33:45.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:33:45.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:33:45.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/ts-2/logs
23:33:45.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/ts-2/data/info.pb
23:33:45.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:33:45.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:33:45.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:33:45.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:33:45.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:33:45.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:33:45.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:33:45.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:33:45.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:33:45.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:33:45.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.37.67:0
23:33:45.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.37.67
23:33:45.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.37.67
23:33:45.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:33:45.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.37.126:46523,127.10.37.125:37371,127.10.37.124:38379
23:33:45.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.37.84:38877
23:33:45.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:33:45.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:33:45.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:33:45.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:45.171015 11018 heartbeater.cc:347] Connected to a master server at 127.10.37.124:38379
23:33:45.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:45.171567 11018 heartbeater.cc:464] Registering TS with master...
23:33:45.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:45.171909 11019 heartbeater.cc:347] Connected to a master server at 127.10.37.125:37371
23:33:45.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:45.172284 11019 heartbeater.cc:464] Registering TS with master...
23:33:45.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:45.173010 11018 heartbeater.cc:511] Master 127.10.37.124:38379 requested a full tablet report, sending...
23:33:45.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:45.173588 11019 heartbeater.cc:511] Master 127.10.37.125:37371 requested a full tablet report, sending...
23:33:45.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:45.173589 11020 heartbeater.cc:347] Connected to a master server at 127.10.37.126:46523
23:33:45.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:45.174225 11020 heartbeater.cc:464] Registering TS with master...
23:33:45.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:45.175073 11020 heartbeater.cc:511] Master 127.10.37.126:46523 requested a full tablet report, sending...
23:33:45.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:45.176489 10610 ts_manager.cc:194] Registered new tserver with Master: 47e975160813450f9addeef8b942eb13 (127.10.37.66:33353)
23:33:45.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:45.177285 10524 ts_manager.cc:194] Registered new tserver with Master: 47e975160813450f9addeef8b942eb13 (127.10.37.66:33353)
23:33:45.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:45.178076 10441 ts_manager.cc:194] Registered new tserver with Master: 47e975160813450f9addeef8b942eb13 (127.10.37.66:33353)
23:33:45.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:45.178892 10610 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.37.66:56015
23:33:45.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:45.504182 11028 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:33:45.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:45.504812 11028 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:33:45.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:45.505558 11028 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:33:45.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:45.537649 11028 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:33:45.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:45.538480 11028 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.37.67
23:33:45.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:45.574353 11028 tablet_server_runner.cc:78] Tablet server non-default flags:
23:33:45.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:33:45.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.37.84:38877
23:33:45.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:33:45.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:33:45.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/ts-2/data
23:33:45.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/ts-2/wal
23:33:45.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:33:45.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:33:45.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.37.67:0
23:33:45.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:33:45.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:33:45.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:33:45.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/ts-2/data/info.pb
23:33:45.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.37.67
23:33:45.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:33:45.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.37.126:46523,127.10.37.125:37371,127.10.37.124:38379
23:33:45.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:33:45.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:33:45.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:33:45.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:33:45.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:33:45.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.37.67
23:33:45.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/ts-2/logs
23:33:45.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:33:45.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:33:45.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
23:33:45.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:33:45.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:33:45.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:33:45.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:33:45.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 20:03:34 UTC on ccycloud.quasar-agiwfo.root.comops.site
23:33:45.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4576543
23:33:45.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:33:45.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:45.575848 11028 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:33:45.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:45.577713 11028 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:33:45.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:45.601388 11038 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:33:46.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:46.183459 11018 heartbeater.cc:503] Master 127.10.37.124:38379 was elected leader, sending a full tablet report...
23:33:46.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:45.602254 11034 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:33:46.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:45.604193 11028 server_base.cc:1034] running on GCE node
23:33:46.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:45.601675 11035 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:33:46.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:46.820212 11028 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:33:46.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:46.822544 11028 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:33:46.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:46.823920 11028 hybrid_clock.cc:648] HybridClock initialized: now 1786318426823887 us; error 46 us; skew 500 ppm
23:33:46.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:46.824771 11028 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:33:46.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:46.831924 11028 webserver.cc:460] Webserver started at http://127.10.37.67:41227/ using document root <none> and password file <none>
23:33:46.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:46.832933 11028 fs_manager.cc:359] Metadata directory not provided
23:33:46.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:46.833140 11028 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:33:46.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:46.833564 11028 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
23:33:46.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:46.838806 11028 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/ts-2/data/instance:
23:33:46.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ac54a89669864e24a43a2e7df003daff"
23:33:46.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 23:33:46 on dist-test-slave-7c35"
23:33:46.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:46.840222 11028 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/ts-2/wal/instance:
23:33:46.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ac54a89669864e24a43a2e7df003daff"
23:33:46.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 23:33:46 on dist-test-slave-7c35"
23:33:46.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:46.848469 11028 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.008s sys 0.000s
23:33:46.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:46.854764 11046 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
23:33:46.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:46.855984 11028 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
23:33:46.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:46.856354 11028 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/ts-2/data,/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/ts-2/wal
23:33:46.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ac54a89669864e24a43a2e7df003daff"
23:33:46.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 23:33:46 on dist-test-slave-7c35"
23:33:46.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:46.856771 11028 fs_report.cc:383] FS layout report
23:33:46.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:33:46.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/ts-2/wal
23:33:46.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/ts-2/wal
23:33:46.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/ts-2/data/data
23:33:46.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:33:46.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:33:46.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:33:46.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:33:46.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:33:46.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:33:46.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:33:46.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:33:46.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:33:46.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:33:46.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:33:46.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:46.899389 11028 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
23:33:46.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:46.901280 11028 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:33:46.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:46.901866 11028 kserver.cc:163] Server-wide thread pool size limit: 3276
23:33:46.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:46.904881 11028 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
23:33:46.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:46.909519 11028 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
23:33:46.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:46.909778 11028 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
23:33:46.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:46.910116 11028 ts_tablet_manager.cc:610] Registered 0 tablets
23:33:46.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:46.910327 11028 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
23:33:47.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:47.076067 11028 rpc_server.cc:307] RPC server started. Bound to: 127.10.37.67:33557
23:33:47.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:47.076181 11168 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.37.67:33557 every 8 connection(s)
23:33:47.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:47.078740 11028 server_base.cc:1166] Dumped server information to /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/ts-2/data/info.pb
23:33:47.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:47.085151 10389 external_mini_cluster.cc:1388] Started /tmp/dist-test-task5zWf6n/build/tsan/bin/kudu as pid 11028
23:33:47.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:47.085760 10389 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster7095652566433661181/ts-2/wal/instance
23:33:47.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:47.136365 11169 heartbeater.cc:347] Connected to a master server at 127.10.37.124:38379
23:33:47.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:47.136938 11169 heartbeater.cc:464] Registering TS with master...
23:33:47.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:47.138142 11171 heartbeater.cc:347] Connected to a master server at 127.10.37.126:46523
23:33:47.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:47.138370 11169 heartbeater.cc:511] Master 127.10.37.124:38379 requested a full tablet report, sending...
23:33:47.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:47.138510 11171 heartbeater.cc:464] Registering TS with master...
23:33:47.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:47.139424 11171 heartbeater.cc:511] Master 127.10.37.126:46523 requested a full tablet report, sending...
23:33:47.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:47.141988 11170 heartbeater.cc:347] Connected to a master server at 127.10.37.125:37371
23:33:47.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:47.142267 11170 heartbeater.cc:464] Registering TS with master...
23:33:47.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:47.142341 10441 ts_manager.cc:194] Registered new tserver with Master: ac54a89669864e24a43a2e7df003daff (127.10.37.67:33557)
23:33:47.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:47.142959 11170 heartbeater.cc:511] Master 127.10.37.125:37371 requested a full tablet report, sending...
23:33:47.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:47.143784 10609 ts_manager.cc:194] Registered new tserver with Master: ac54a89669864e24a43a2e7df003daff (127.10.37.67:33557)
23:33:47.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:47.145494 10609 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.37.67:42991
23:33:47.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:47.145797 10523 ts_manager.cc:194] Registered new tserver with Master: ac54a89669864e24a43a2e7df003daff (127.10.37.67:33557)
23:33:47.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:47.157670 10389 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
23:33:47.169 [DEBUG - main] (MiniKuduCluster.java:195) Response:
23:33:47.171 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
23:33:47.182 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.10.37.126"
port: 46523
}
bound_http_address {
host: "127.10.37.126"
port: 36495
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.10.37.125"
port: 37371
}
bound_http_address {
host: "127.10.37.125"
port: 43661
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.10.37.124"
port: 38379
}
bound_http_address {
host: "127.10.37.124"
port: 33793
}
}
}
23:33:47.189 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
23:33:47.195 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.10.37.65"
port: 43627
}
bound_http_address {
host: "127.10.37.65"
port: 36571
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.10.37.66"
port: 33353
}
bound_http_address {
host: "127.10.37.66"
port: 42827
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.10.37.67"
port: 33557
}
bound_http_address {
host: "127.10.37.67"
port: 41227
}
}
}
23:33:47.196 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
23:33:47.400 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
23:33:47.530 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task5zWf6n/test-tmp
23:33:47.531 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
23:33:47.531 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task5zWf6n/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
23:33:47.534 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
23:33:47.895 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
23:33:47.895 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
23:33:47.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:47.898718 11181 mini_oidc.cc:152] Starting JWKS server
23:33:47.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:47.902889 11181 webserver.cc:460] Webserver started at http://127.0.0.1:45609/ using document root <none> and password file <none>
23:33:47.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:47.903414 11181 mini_oidc.cc:189] Starting OIDC Discovery server
23:33:47.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:47.905289 11181 webserver.cc:460] Webserver started at http://127.0.0.1:36987/ using document root <none> and password file <none>
23:33:47.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
23:33:47.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/krb5kdc/principal' for realm 'KRBTEST.COM',
23:33:47.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
23:33:48.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 23:33:47 dist-test-slave-7c35 krb5kdc[11195](info): setting up network...
23:33:48.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
23:33:48.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 23:33:47 dist-test-slave-7c35 krb5kdc[11195](info): set up 2 sockets
23:33:48.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 23:33:47 dist-test-slave-7c35 krb5kdc[11195](info): commencing operation
23:33:48.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
23:33:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:48.149926 11169 heartbeater.cc:503] Master 127.10.37.124:38379 was elected leader, sending a full tablet report...
23:33:49.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:49.183750 11181 mini_kdc.cc:121] Time spent starting KDC: real 1.251s user 0.000s sys 0.031s
23:33:49.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
23:33:49.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
23:33:49.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
23:33:49.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
23:33:49.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
23:33:49.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
23:33:49.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
23:33:49.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
23:33:49.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
23:33:49.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
23:33:49.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/krb5kdc/test-user.keytab.
23:33:49.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/krb5kdc/test-user.keytab.
23:33:49.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 23:33:49 dist-test-slave-7c35 krb5kdc[11195](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786318429, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
23:33:49.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
23:33:49.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-09T23:33:49Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
23:33:49.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-09T23:33:49Z Disabled control of system clock
23:33:49.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.10.235.126@KRBTEST.COM; defaulting to no policy
23:33:49.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
23:33:49.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.10.235.126@KRBTEST.COM" created.
23:33:49.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
23:33:49.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.10.235.126 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/krb5kdc/kudu_127.10.235.126.keytab.
23:33:49.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.10.235.126 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/krb5kdc/kudu_127.10.235.126.keytab.
23:33:49.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:49.432457 11181 external_mini_cluster.cc:1326] Running /tmp/dist-test-task5zWf6n/build/tsan/bin/kudu
23:33:49.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task5zWf6n/build/tsan/bin/kudu
23:33:49.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-0/wal
23:33:49.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-0/data
23:33:49.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:33:49.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:33:49.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:33:49.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:33:49.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:33:49.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:33:49.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-0/logs
23:33:49.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-0/data/info.pb
23:33:49.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:33:49.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:33:49.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:33:49.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:33:49.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:33:49.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:33:49.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:33:49.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:33:49.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
23:33:49.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:33:49.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:33:49.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:33:49.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.235.126:46281
23:33:49.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.235.126
23:33:49.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:33:49.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/krb5kdc/kudu_127.10.235.126.keytab
23:33:49.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.10.235.126
23:33:49.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
23:33:49.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
23:33:49.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
23:33:49.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.235.84:45015
23:33:49.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:33:49.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:33:49.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:33:49.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:33:49.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.235.126:46281,127.10.235.125:45571,127.10.235.124:34231
23:33:49.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:33:49.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45609/jwks.json
23:33:49.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
23:33:49.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:49.767961 11208 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:33:49.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:49.768543 11208 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:33:49.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:49.769133 11208 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:33:49.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:49.800907 11208 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
23:33:49.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:49.801162 11208 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
23:33:49.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:49.801424 11208 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:33:49.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:49.801731 11208 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
23:33:49.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:49.801935 11208 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
23:33:49.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:49.802130 11208 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
23:33:49.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:49.802268 11208 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45609/jwks.json
23:33:49.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:49.840787 11208 master_runner.cc:386] Master server non-default flags:
23:33:49.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:33:49.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.235.84:45015
23:33:49.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:33:49.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:33:49.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-0/data
23:33:49.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-0/wal
23:33:49.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
23:33:49.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:33:49.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.235.126:46281,127.10.235.125:45571,127.10.235.124:34231
23:33:49.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/krb5kdc/kudu_127.10.235.126.keytab
23:33:49.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.10.235.126
23:33:49.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:33:49.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:33:49.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:33:49.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.235.126:46281
23:33:49.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:33:49.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:33:49.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:33:49.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45609/jwks.json
23:33:49.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
23:33:49.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
23:33:49.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
23:33:49.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:33:49.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:33:49.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-0/data/info.pb
23:33:49.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.235.126
23:33:49.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:33:49.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:33:49.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:33:49.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:33:49.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:33:49.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:33:49.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-0/logs
23:33:49.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:33:49.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:33:49.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
23:33:49.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
23:33:49.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:33:49.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:33:49.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:33:49.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 20:03:34 UTC on ccycloud.quasar-agiwfo.root.comops.site
23:33:49.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4576543
23:33:49.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:33:49.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:49.842073 11208 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:33:49.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:49.843182 11208 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:33:49.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:49.863059 11216 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:33:51.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:49.863468 11218 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:33:51.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:49.863867 11215 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:33:51.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:49.864306 11208 server_base.cc:1034] running on GCE node
23:33:51.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:51.128566 11208 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:33:51.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:51.131556 11208 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:33:51.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:51.132938 11208 hybrid_clock.cc:648] HybridClock initialized: now 1786318431132905 us; error 73 us; skew 500 ppm
23:33:51.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:51.133756 11208 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:33:51.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 23:33:51 dist-test-slave-7c35 krb5kdc[11195](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786318431, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.235.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
23:33:51.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:51.139856 11208 init.cc:377] Logged in from keytab as kudu/127.10.235.126@KRBTEST.COM (short username kudu)
23:33:51.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:51.145561 11208 webserver.cc:460] Webserver started at http://127.10.235.126:44905/ using document root <none> and password file <none>
23:33:51.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:51.146504 11208 fs_manager.cc:359] Metadata directory not provided
23:33:51.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:51.146690 11208 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:33:51.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:51.147172 11208 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
23:33:51.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:51.152498 11208 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-0/data/instance:
23:33:51.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ff38ce53b3054dd4a6aa20ba37c685bc"
23:33:51.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 23:33:51 on dist-test-slave-7c35"
23:33:51.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:51.153723 11208 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-0/wal/instance:
23:33:51.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ff38ce53b3054dd4a6aa20ba37c685bc"
23:33:51.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 23:33:51 on dist-test-slave-7c35"
23:33:51.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:51.163313 11208 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.011s sys 0.000s
23:33:51.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:51.169550 11229 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
23:33:51.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:51.170817 11208 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.006s sys 0.000s
23:33:51.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:51.171322 11208 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-0/data,/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-0/wal
23:33:51.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ff38ce53b3054dd4a6aa20ba37c685bc"
23:33:51.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 23:33:51 on dist-test-slave-7c35"
23:33:51.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:51.171741 11208 fs_report.cc:383] FS layout report
23:33:51.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:33:51.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-0/wal
23:33:51.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-0/wal
23:33:51.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-0/data/data
23:33:51.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:33:51.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:33:51.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:33:51.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:33:51.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:33:51.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:33:51.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:33:51.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:33:51.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:33:51.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:33:51.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:33:51.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:51.213310 11208 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
23:33:51.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:51.220278 11208 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:33:51.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:51.220871 11208 kserver.cc:163] Server-wide thread pool size limit: 3276
23:33:51.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:51.306079 11208 rpc_server.cc:307] RPC server started. Bound to: 127.10.235.126:46281
23:33:51.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:51.306216 11282 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.235.126:46281 every 8 connection(s)
23:33:51.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:51.309288 11208 server_base.cc:1166] Dumped server information to /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-0/data/info.pb
23:33:51.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:51.316602 11283 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
23:33:51.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:51.317423 11181 external_mini_cluster.cc:1388] Started /tmp/dist-test-task5zWf6n/build/tsan/bin/kudu as pid 11208
23:33:51.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:51.317817 11181 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-0/wal/instance
23:33:51.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:51.329197 11283 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.235.126" port: 46281 } has no permanent_uuid. Determining permanent_uuid...
23:33:51.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.10.235.125@KRBTEST.COM; defaulting to no policy
23:33:51.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
23:33:51.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.10.235.125@KRBTEST.COM" created.
23:33:51.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 23:33:51 dist-test-slave-7c35 krb5kdc[11195](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786318431, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.235.126@KRBTEST.COM for kudu/127.10.235.126@KRBTEST.COM
23:33:51.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:51.384115 11283 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.235.125" port: 45571 } has no permanent_uuid. Determining permanent_uuid...
23:33:51.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
23:33:51.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.10.235.125 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/krb5kdc/kudu_127.10.235.125.keytab.
23:33:51.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.10.235.125 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/krb5kdc/kudu_127.10.235.125.keytab.
23:33:51.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:51.386379 11231 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.235.125:45571: connect: Connection refused (error 111)
23:33:51.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:51.387612 11181 external_mini_cluster.cc:1326] Running /tmp/dist-test-task5zWf6n/build/tsan/bin/kudu
23:33:51.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task5zWf6n/build/tsan/bin/kudu
23:33:51.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-1/wal
23:33:51.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-1/data
23:33:51.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:33:51.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:33:51.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:33:51.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:33:51.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:33:51.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:33:51.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-1/logs
23:33:51.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-1/data/info.pb
23:33:51.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:33:51.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:33:51.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:33:51.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:33:51.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:33:51.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:33:51.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:33:51.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:33:51.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
23:33:51.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:33:51.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:33:51.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:33:51.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.235.125:45571
23:33:51.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.235.125
23:33:51.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:33:51.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/krb5kdc/kudu_127.10.235.125.keytab
23:33:51.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.10.235.125
23:33:51.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
23:33:51.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
23:33:51.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
23:33:51.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.235.84:45015
23:33:51.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:33:51.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:33:51.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:33:51.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:33:51.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.235.126:46281,127.10.235.125:45571,127.10.235.124:34231
23:33:51.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:33:51.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45609/jwks.json
23:33:51.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
23:33:51.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:51.390937 11283 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.235.125:45571: Network error: Client connection negotiation failed: client connection to 127.10.235.125:45571: connect: Connection refused (error 111)
23:33:51.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:51.440557 11283 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.235.125" port: 45571 } attempt: 1
23:33:51.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:51.444577 11283 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.235.125:45571: Network error: Client connection negotiation failed: client connection to 127.10.235.125:45571: connect: Connection refused (error 111)
23:33:51.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:51.513041 11283 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.235.125" port: 45571 } attempt: 2
23:33:51.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:51.517558 11283 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.235.125:45571: Network error: Client connection negotiation failed: client connection to 127.10.235.125:45571: connect: Connection refused (error 111)
23:33:51.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:51.609066 11283 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.235.125" port: 45571 } attempt: 3
23:33:51.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:51.613523 11283 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.235.125:45571: Network error: Client connection negotiation failed: client connection to 127.10.235.125:45571: connect: Connection refused (error 111)
23:33:51.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:51.744071 11290 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:33:51.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:51.744789 11290 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:33:51.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:51.745862 11290 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:33:51.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:51.757014 11283 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.235.125" port: 45571 } attempt: 4
23:33:51.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:51.761153 11283 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.235.125:45571: Network error: Client connection negotiation failed: client connection to 127.10.235.125:45571: connect: Connection refused (error 111)
23:33:51.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:51.779449 11290 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
23:33:51.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:51.779748 11290 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
23:33:51.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:51.780138 11290 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:33:51.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:51.780484 11290 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
23:33:51.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:51.780920 11290 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
23:33:51.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:51.781140 11290 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
23:33:51.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:51.781281 11290 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45609/jwks.json
23:33:51.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:51.819144 11290 master_runner.cc:386] Master server non-default flags:
23:33:51.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:33:51.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.235.84:45015
23:33:51.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:33:51.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:33:51.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-1/data
23:33:51.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-1/wal
23:33:51.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
23:33:51.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:33:51.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.235.126:46281,127.10.235.125:45571,127.10.235.124:34231
23:33:51.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/krb5kdc/kudu_127.10.235.125.keytab
23:33:51.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.10.235.125
23:33:51.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:33:51.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:33:51.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:33:51.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.235.125:45571
23:33:51.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:33:51.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:33:51.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:33:51.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45609/jwks.json
23:33:51.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
23:33:51.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
23:33:51.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
23:33:51.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:33:51.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:33:51.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-1/data/info.pb
23:33:51.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.235.125
23:33:51.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:33:51.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:33:51.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:33:51.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:33:51.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:33:51.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:33:51.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-1/logs
23:33:51.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:33:51.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:33:51.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
23:33:51.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
23:33:51.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:33:51.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:33:51.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:33:51.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 20:03:34 UTC on ccycloud.quasar-agiwfo.root.comops.site
23:33:51.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4576543
23:33:51.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:33:51.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:51.820483 11290 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:33:51.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:51.821765 11290 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:33:51.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:51.841277 11299 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:33:52.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:52.060600 11283 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.235.125" port: 45571 } attempt: 5
23:33:52.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:52.065649 11283 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.235.125:45571: Network error: Client connection negotiation failed: client connection to 127.10.235.125:45571: connect: Connection refused (error 111)
23:33:52.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:52.613092 11283 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.235.125" port: 45571 } attempt: 6
23:33:52.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:52.621332 11283 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.235.125:45571: Network error: Client connection negotiation failed: client connection to 127.10.235.125:45571: connect: Connection refused (error 111)
23:33:53.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:51.842823 11290 server_base.cc:1034] running on GCE node
23:33:53.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:51.842478 11296 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:33:53.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:51.841308 11297 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:33:53.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:53.123220 11290 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:33:53.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:53.126674 11290 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:33:53.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:53.128199 11290 hybrid_clock.cc:648] HybridClock initialized: now 1786318433128168 us; error 41 us; skew 500 ppm
23:33:53.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:53.129127 11290 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:33:53.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 23:33:53 dist-test-slave-7c35 krb5kdc[11195](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786318433, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.235.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
23:33:53.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:53.140501 11290 init.cc:377] Logged in from keytab as kudu/127.10.235.125@KRBTEST.COM (short username kudu)
23:33:53.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:53.146936 11290 webserver.cc:460] Webserver started at http://127.10.235.125:34077/ using document root <none> and password file <none>
23:33:53.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:53.148164 11290 fs_manager.cc:359] Metadata directory not provided
23:33:53.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:53.148363 11290 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:33:53.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:53.148766 11290 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
23:33:53.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:53.153982 11290 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-1/data/instance:
23:33:53.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ae4c1992659c4ec78bb1e9066aa24096"
23:33:53.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 23:33:53 on dist-test-slave-7c35"
23:33:53.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:53.155365 11290 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-1/wal/instance:
23:33:53.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ae4c1992659c4ec78bb1e9066aa24096"
23:33:53.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 23:33:53 on dist-test-slave-7c35"
23:33:53.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:53.163988 11290 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.006s sys 0.001s
23:33:53.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:53.171620 11310 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
23:33:53.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:53.173063 11290 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.001s sys 0.003s
23:33:53.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:53.173486 11290 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-1/data,/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-1/wal
23:33:53.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ae4c1992659c4ec78bb1e9066aa24096"
23:33:53.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 23:33:53 on dist-test-slave-7c35"
23:33:53.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:53.173893 11290 fs_report.cc:383] FS layout report
23:33:53.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:33:53.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-1/wal
23:33:53.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-1/wal
23:33:53.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-1/data/data
23:33:53.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:33:53.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:33:53.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:33:53.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:33:53.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:33:53.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:33:53.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:33:53.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:33:53.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:33:53.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:33:53.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:33:53.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:53.245127 11290 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
23:33:53.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:53.250965 11290 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:33:53.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:53.251507 11290 kserver.cc:163] Server-wide thread pool size limit: 3276
23:33:53.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:53.345345 11290 rpc_server.cc:307] RPC server started. Bound to: 127.10.235.125:45571
23:33:53.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:53.345440 11368 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.235.125:45571 every 8 connection(s)
23:33:53.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:53.348651 11290 server_base.cc:1166] Dumped server information to /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-1/data/info.pb
23:33:53.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:53.354759 11369 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
23:33:53.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:53.357867 11181 external_mini_cluster.cc:1388] Started /tmp/dist-test-task5zWf6n/build/tsan/bin/kudu as pid 11290
23:33:53.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:53.358260 11181 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-1/wal/instance
23:33:53.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:53.367481 11369 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.235.126" port: 46281 } has no permanent_uuid. Determining permanent_uuid...
23:33:53.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.10.235.124@KRBTEST.COM; defaulting to no policy
23:33:53.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
23:33:53.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.10.235.124@KRBTEST.COM" created.
23:33:53.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 23:33:53 dist-test-slave-7c35 krb5kdc[11195](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786318433, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.235.125@KRBTEST.COM for kudu/127.10.235.126@KRBTEST.COM
23:33:53.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:53.412708 11369 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.235.125" port: 45571 } has no permanent_uuid. Determining permanent_uuid...
23:33:53.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
23:33:53.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.10.235.124 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/krb5kdc/kudu_127.10.235.124.keytab.
23:33:53.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.10.235.124 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/krb5kdc/kudu_127.10.235.124.keytab.
23:33:53.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:53.418942 11181 external_mini_cluster.cc:1326] Running /tmp/dist-test-task5zWf6n/build/tsan/bin/kudu
23:33:53.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task5zWf6n/build/tsan/bin/kudu
23:33:53.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-2/wal
23:33:53.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-2/data
23:33:53.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:33:53.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:33:53.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:33:53.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:33:53.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:33:53.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:33:53.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-2/logs
23:33:53.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-2/data/info.pb
23:33:53.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:33:53.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:33:53.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:33:53.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:33:53.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:33:53.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:33:53.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:33:53.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:33:53.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
23:33:53.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:33:53.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:33:53.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:33:53.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.235.124:34231
23:33:53.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.235.124
23:33:53.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:33:53.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/krb5kdc/kudu_127.10.235.124.keytab
23:33:53.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.10.235.124
23:33:53.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
23:33:53.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
23:33:53.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
23:33:53.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.235.84:45015
23:33:53.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:33:53.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:33:53.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:33:53.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:33:53.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.235.126:46281,127.10.235.125:45571,127.10.235.124:34231
23:33:53.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:33:53.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45609/jwks.json
23:33:53.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
23:33:53.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 23:33:53 dist-test-slave-7c35 krb5kdc[11195](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786318433, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.235.125@KRBTEST.COM for kudu/127.10.235.125@KRBTEST.COM
23:33:53.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:53.449393 11369 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.235.124" port: 34231 } has no permanent_uuid. Determining permanent_uuid...
23:33:53.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:53.451776 11314 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.235.124:34231: connect: Connection refused (error 111)
23:33:53.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:53.456566 11369 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.235.124:34231: Network error: Client connection negotiation failed: client connection to 127.10.235.124:34231: connect: Connection refused (error 111)
23:33:53.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:53.506098 11369 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.235.124" port: 34231 } attempt: 1
23:33:53.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:53.510545 11369 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.235.124:34231: Network error: Client connection negotiation failed: client connection to 127.10.235.124:34231: connect: Connection refused (error 111)
23:33:53.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:53.579039 11369 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.235.124" port: 34231 } attempt: 2
23:33:53.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:53.583719 11369 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.235.124:34231: Network error: Client connection negotiation failed: client connection to 127.10.235.124:34231: connect: Connection refused (error 111)
23:33:53.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:53.675256 11369 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.235.124" port: 34231 } attempt: 3
23:33:53.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:53.679908 11369 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.235.124:34231: Network error: Client connection negotiation failed: client connection to 127.10.235.124:34231: connect: Connection refused (error 111)
23:33:53.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:53.681808 11283 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.235.125" port: 45571 } attempt: 7
23:33:53.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 23:33:53 dist-test-slave-7c35 krb5kdc[11195](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786318431, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.235.126@KRBTEST.COM for kudu/127.10.235.125@KRBTEST.COM
23:33:53.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:53.710207 11283 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.235.124" port: 34231 } has no permanent_uuid. Determining permanent_uuid...
23:33:53.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:53.716097 11283 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.235.124:34231: Network error: Client connection negotiation failed: client connection to 127.10.235.124:34231: connect: Connection refused (error 111)
23:33:53.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:53.774565 11283 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.235.124" port: 34231 } attempt: 1
23:33:53.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:53.778875 11283 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.235.124:34231: Network error: Client connection negotiation failed: client connection to 127.10.235.124:34231: connect: Connection refused (error 111)
23:33:53.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:53.798198 11376 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:33:53.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:53.798961 11376 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:33:53.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:53.799712 11376 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:33:53.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:53.823342 11369 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.235.124" port: 34231 } attempt: 4
23:33:53.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:53.827622 11369 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.235.124:34231: Network error: Client connection negotiation failed: client connection to 127.10.235.124:34231: connect: Connection refused (error 111)
23:33:53.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:53.835707 11376 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
23:33:53.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:53.836045 11376 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
23:33:53.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:53.836469 11376 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:33:53.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:53.836828 11376 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
23:33:53.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:53.837136 11376 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
23:33:53.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:53.837441 11376 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
23:33:53.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:53.837671 11376 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45609/jwks.json
23:33:53.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:53.860265 11283 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.235.124" port: 34231 } attempt: 2
23:33:53.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:53.864398 11283 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.235.124:34231: Network error: Client connection negotiation failed: client connection to 127.10.235.124:34231: connect: Connection refused (error 111)
23:33:53.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:53.877866 11376 master_runner.cc:386] Master server non-default flags:
23:33:53.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:33:53.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.235.84:45015
23:33:53.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:33:53.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:33:53.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-2/data
23:33:53.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-2/wal
23:33:53.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
23:33:53.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:33:53.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.235.126:46281,127.10.235.125:45571,127.10.235.124:34231
23:33:53.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/krb5kdc/kudu_127.10.235.124.keytab
23:33:53.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.10.235.124
23:33:53.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:33:53.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:33:53.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:33:53.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.235.124:34231
23:33:53.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:33:53.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:33:53.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:33:53.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45609/jwks.json
23:33:53.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
23:33:53.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
23:33:53.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
23:33:53.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:33:53.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:33:53.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-2/data/info.pb
23:33:53.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.235.124
23:33:53.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:33:53.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:33:53.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:33:53.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:33:53.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:33:53.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:33:53.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-2/logs
23:33:53.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:33:53.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:33:53.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
23:33:53.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
23:33:53.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:33:53.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:33:53.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:33:53.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 20:03:34 UTC on ccycloud.quasar-agiwfo.root.comops.site
23:33:53.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4576543
23:33:53.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:33:53.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:53.879192 11376 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:33:53.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:53.880173 11376 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:33:53.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:53.900357 11388 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:33:53.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:53.949865 11283 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.235.124" port: 34231 } attempt: 3
23:33:53.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:53.954909 11283 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.235.124:34231: Network error: Client connection negotiation failed: client connection to 127.10.235.124:34231: connect: Connection refused (error 111)
23:33:54.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:54.095328 11283 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.235.124" port: 34231 } attempt: 4
23:33:54.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:54.099478 11283 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.235.124:34231: Network error: Client connection negotiation failed: client connection to 127.10.235.124:34231: connect: Connection refused (error 111)
23:33:54.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:54.127030 11369 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.235.124" port: 34231 } attempt: 5
23:33:54.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:54.131558 11369 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.235.124:34231: Network error: Client connection negotiation failed: client connection to 127.10.235.124:34231: connect: Connection refused (error 111)
23:33:54.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:54.383082 11283 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.235.124" port: 34231 } attempt: 5
23:33:54.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:54.387583 11283 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.235.124:34231: Network error: Client connection negotiation failed: client connection to 127.10.235.124:34231: connect: Connection refused (error 111)
23:33:54.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:54.679021 11369 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.235.124" port: 34231 } attempt: 6
23:33:54.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:54.686487 11369 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.235.124:34231: Network error: Client connection negotiation failed: client connection to 127.10.235.124:34231: connect: Connection refused (error 111)
23:33:54.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:54.940274 11283 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.235.124" port: 34231 } attempt: 6
23:33:54.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:54.947706 11283 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.235.124:34231: Network error: Client connection negotiation failed: client connection to 127.10.235.124:34231: connect: Connection refused (error 111)
23:33:55.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:53.900866 11389 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:33:55.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:53.901397 11391 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:33:55.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:53.903249 11376 server_base.cc:1034] running on GCE node
23:33:55.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:55.153781 11376 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:33:55.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:55.157169 11376 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:33:55.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:55.158658 11376 hybrid_clock.cc:648] HybridClock initialized: now 1786318435158619 us; error 56 us; skew 500 ppm
23:33:55.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:55.159629 11376 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:33:55.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 23:33:55 dist-test-slave-7c35 krb5kdc[11195](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786318435, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.235.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
23:33:55.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:55.165920 11376 init.cc:377] Logged in from keytab as kudu/127.10.235.124@KRBTEST.COM (short username kudu)
23:33:55.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:55.172246 11376 webserver.cc:460] Webserver started at http://127.10.235.124:45773/ using document root <none> and password file <none>
23:33:55.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:55.173290 11376 fs_manager.cc:359] Metadata directory not provided
23:33:55.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:55.173518 11376 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:33:55.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:55.174012 11376 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
23:33:55.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:55.180449 11376 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-2/data/instance:
23:33:55.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1c0deac92b4c497e92c36e0e1f2b75ac"
23:33:55.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 23:33:55 on dist-test-slave-7c35"
23:33:55.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:55.182585 11376 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-2/wal/instance:
23:33:55.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1c0deac92b4c497e92c36e0e1f2b75ac"
23:33:55.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 23:33:55 on dist-test-slave-7c35"
23:33:55.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:55.192312 11376 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.008s sys 0.000s
23:33:55.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:55.198629 11404 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
23:33:55.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:55.199826 11376 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
23:33:55.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:55.200196 11376 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-2/data,/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-2/wal
23:33:55.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1c0deac92b4c497e92c36e0e1f2b75ac"
23:33:55.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 23:33:55 on dist-test-slave-7c35"
23:33:55.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:55.200598 11376 fs_report.cc:383] FS layout report
23:33:55.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:33:55.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-2/wal
23:33:55.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-2/wal
23:33:55.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-2/data/data
23:33:55.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:33:55.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:33:55.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:33:55.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:33:55.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:33:55.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:33:55.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:33:55.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:33:55.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:33:55.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:33:55.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:33:55.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:55.268270 11376 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
23:33:55.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:55.274739 11376 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:33:55.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:55.275252 11376 kserver.cc:163] Server-wide thread pool size limit: 3276
23:33:55.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:55.368809 11376 rpc_server.cc:307] RPC server started. Bound to: 127.10.235.124:34231
23:33:55.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:55.368897 11460 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.235.124:34231 every 8 connection(s)
23:33:55.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:55.372133 11376 server_base.cc:1166] Dumped server information to /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-2/data/info.pb
23:33:55.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:55.375385 11181 external_mini_cluster.cc:1388] Started /tmp/dist-test-task5zWf6n/build/tsan/bin/kudu as pid 11376
23:33:55.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:55.376024 11181 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-2/wal/instance
23:33:55.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:55.378597 11462 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
23:33:55.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:55.393265 11462 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.235.126" port: 46281 } has no permanent_uuid. Determining permanent_uuid...
23:33:55.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 23:33:55 dist-test-slave-7c35 krb5kdc[11195](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786318429, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.10.235.124@KRBTEST.COM
23:33:55.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 23:33:55 dist-test-slave-7c35 krb5kdc[11195](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786318435, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.235.124@KRBTEST.COM for kudu/127.10.235.126@KRBTEST.COM
23:33:55.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:55.428762 11462 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.235.125" port: 45571 } has no permanent_uuid. Determining permanent_uuid...
23:33:55.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 23:33:55 dist-test-slave-7c35 krb5kdc[11195](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786318435, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.235.124@KRBTEST.COM for kudu/127.10.235.125@KRBTEST.COM
23:33:55.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 23:33:55 dist-test-slave-7c35 krb5kdc[11195](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786318429, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.10.235.125@KRBTEST.COM
23:33:55.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:55.462105 11462 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.235.124" port: 34231 } has no permanent_uuid. Determining permanent_uuid...
23:33:55.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 23:33:55 dist-test-slave-7c35 krb5kdc[11195](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786318429, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.10.235.126@KRBTEST.COM
23:33:55.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 23:33:55 dist-test-slave-7c35 krb5kdc[11195](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786318435, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.235.124@KRBTEST.COM for kudu/127.10.235.124@KRBTEST.COM
23:33:55.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:55.499045 11181 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
23:33:55.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:55.509124 11462 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac: Bootstrap starting.
23:33:55.510 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
23:33:55.512 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
23:33:55.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:55.515903 11462 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac: Neither blocks nor log segments found. Creating new log.
23:33:55.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:55.517834 11462 log.cc:824] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac: Log is configured to *not* fsync() on all Append() calls
23:33:55.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:55.523142 11462 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac: No bootstrap required, opened a new log
23:33:55.524 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
23:33:55.524 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
23:33:55.527 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.10.235.126"
port: 46281
}
bound_http_address {
host: "127.10.235.126"
port: 44905
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.10.235.125"
port: 45571
}
bound_http_address {
host: "127.10.235.125"
port: 34077
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.10.235.124"
port: 34231
}
bound_http_address {
host: "127.10.235.124"
port: 45773
}
}
}
23:33:55.528 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
23:33:55.528 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
23:33:55.528 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
23:33:55.530 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
23:33:55.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:55.553943 11462 raft_consensus.cc:348] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ff38ce53b3054dd4a6aa20ba37c685bc" member_type: VOTER last_known_addr { host: "127.10.235.126" port: 46281 } } peers { permanent_uuid: "ae4c1992659c4ec78bb1e9066aa24096" member_type: VOTER last_known_addr { host: "127.10.235.125" port: 45571 } } peers { permanent_uuid: "1c0deac92b4c497e92c36e0e1f2b75ac" member_type: VOTER last_known_addr { host: "127.10.235.124" port: 34231 } }
23:33:55.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:55.554562 11462 raft_consensus.cc:374] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
23:33:55.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:55.554755 11462 raft_consensus.cc:729] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1c0deac92b4c497e92c36e0e1f2b75ac, State: Initialized, Role: FOLLOWER
23:33:55.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:55.555549 11462 consensus_queue.cc:260] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ff38ce53b3054dd4a6aa20ba37c685bc" member_type: VOTER last_known_addr { host: "127.10.235.126" port: 46281 } } peers { permanent_uuid: "ae4c1992659c4ec78bb1e9066aa24096" member_type: VOTER last_known_addr { host: "127.10.235.125" port: 45571 } } peers { permanent_uuid: "1c0deac92b4c497e92c36e0e1f2b75ac" member_type: VOTER last_known_addr { host: "127.10.235.124" port: 34231 } }
23:33:55.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:55.558787 11474 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ff38ce53b3054dd4a6aa20ba37c685bc" member_type: VOTER last_known_addr { host: "127.10.235.126" port: 46281 } } peers { permanent_uuid: "ae4c1992659c4ec78bb1e9066aa24096" member_type: VOTER last_known_addr { host: "127.10.235.125" port: 45571 } } peers { permanent_uuid: "1c0deac92b4c497e92c36e0e1f2b75ac" member_type: VOTER last_known_addr { host: "127.10.235.124" port: 34231 } } }
23:33:55.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:55.560036 11462 sys_catalog.cc:564] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [sys.catalog]: configured and running, proceeding with master startup.
23:33:55.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:55.560199 11474 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [sys.catalog]: This master's current role is: FOLLOWER
23:33:55.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 23:33:55 dist-test-slave-7c35 krb5kdc[11195](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786318435, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
23:33:55.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
23:33:55.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:55.580015 11474 raft_consensus.cc:482] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
23:33:55.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:55.580575 11474 raft_consensus.cc:504] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ff38ce53b3054dd4a6aa20ba37c685bc" member_type: VOTER last_known_addr { host: "127.10.235.126" port: 46281 } } peers { permanent_uuid: "ae4c1992659c4ec78bb1e9066aa24096" member_type: VOTER last_known_addr { host: "127.10.235.125" port: 45571 } } peers { permanent_uuid: "1c0deac92b4c497e92c36e0e1f2b75ac" member_type: VOTER last_known_addr { host: "127.10.235.124" port: 34231 } }
23:33:55.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:55.584195 11474 leader_election.cc:288] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ff38ce53b3054dd4a6aa20ba37c685bc (127.10.235.126:46281), ae4c1992659c4ec78bb1e9066aa24096 (127.10.235.125:45571)
23:33:55.586 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
23:33:55.589 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/krb5kdc/krb5cc
23:33:55.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:55.595806 11486 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:33:55.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:55.596462 11486 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:33:55.602 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
23:33:55.603 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/krb5kdc/krb5cc
23:33:55.605 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
23:33:55.610 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
23:33:55.612 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
23:33:55.613 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
23:33:55.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:55.588546 11376 master_runner.cc:186] Error getting master registration for 127.10.235.126:46281: OK, instance_id { permanent_uuid: "ff38ce53b3054dd4a6aa20ba37c685bc" instance_seqno: 1786318431281777 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
23:33:55.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:55.626423 11376 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
23:33:55.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 23:33:55 dist-test-slave-7c35 krb5kdc[11195](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786318435, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
23:33:55.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
23:33:55.678 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
23:33:55.678 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/krb5kdc/krb5cc
23:33:55.680 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
23:33:55.681 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.10.235.125:45571
23:33:55.683 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
23:33:55.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:55.684333 11181 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task5zWf6n/build/tsan/bin/kudu with pid 11290
23:33:55.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:55.715983 11406 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv got EOF from 127.10.235.125:45571 (error 108)
23:33:55.717 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
23:33:55.717 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.10.235.126:46281
23:33:55.718 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
23:33:55.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:55.718410 11181 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task5zWf6n/build/tsan/bin/kudu with pid 11208
23:33:55.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:55.731040 11406 leader_election.cc:334] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ae4c1992659c4ec78bb1e9066aa24096 (127.10.235.125:45571): Network error: Client connection negotiation failed: client connection to 127.10.235.125:45571: connect: Connection refused (error 111)
23:33:55.754 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
23:33:55.755 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.10.235.124:34231
23:33:55.755 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
23:33:55.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:55.755790 11181 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task5zWf6n/build/tsan/bin/kudu with pid 11376
23:33:55.784 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
23:33:55.784 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.10.235.125:45571
23:33:55.786 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
23:33:55.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:55.788081 11181 external_mini_cluster.cc:1326] Running /tmp/dist-test-task5zWf6n/build/tsan/bin/kudu
23:33:55.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task5zWf6n/build/tsan/bin/kudu
23:33:55.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-1/wal
23:33:55.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-1/data
23:33:55.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:33:55.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:33:55.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:33:55.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:33:55.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:33:55.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:33:55.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-1/logs
23:33:55.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-1/data/info.pb
23:33:55.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:33:55.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:33:55.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:33:55.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:33:55.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:33:55.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:33:55.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:33:55.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:33:55.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
23:33:55.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:33:55.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:33:55.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:33:55.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.235.125:45571
23:33:55.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.235.125
23:33:55.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34077
23:33:55.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/krb5kdc/kudu_127.10.235.125.keytab
23:33:55.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.10.235.125
23:33:55.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
23:33:55.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
23:33:55.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
23:33:55.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.235.84:45015
23:33:55.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:33:55.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:33:55.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:33:55.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:33:55.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.235.126:46281,127.10.235.125:45571,127.10.235.124:34231
23:33:55.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:33:55.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45609/jwks.json
23:33:55.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
23:33:56.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:56.140872 11491 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:33:56.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:56.141629 11491 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:33:56.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:56.142406 11491 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:33:56.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:56.176010 11491 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
23:33:56.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:56.176328 11491 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
23:33:56.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:56.176764 11491 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:33:56.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:56.177124 11491 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
23:33:56.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:56.177449 11491 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
23:33:56.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:56.177743 11491 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
23:33:56.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:56.177953 11491 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45609/jwks.json
23:33:56.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:56.221438 11491 master_runner.cc:386] Master server non-default flags:
23:33:56.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:33:56.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.235.84:45015
23:33:56.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:33:56.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:33:56.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-1/data
23:33:56.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-1/wal
23:33:56.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
23:33:56.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:33:56.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.235.126:46281,127.10.235.125:45571,127.10.235.124:34231
23:33:56.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/krb5kdc/kudu_127.10.235.125.keytab
23:33:56.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.10.235.125
23:33:56.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:33:56.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:33:56.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:33:56.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.235.125:45571
23:33:56.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:33:56.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:33:56.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:33:56.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45609/jwks.json
23:33:56.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
23:33:56.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
23:33:56.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
23:33:56.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:33:56.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:33:56.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-1/data/info.pb
23:33:56.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.235.125
23:33:56.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34077
23:33:56.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:33:56.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:33:56.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:33:56.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:33:56.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:33:56.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-1/logs
23:33:56.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:33:56.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:33:56.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
23:33:56.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
23:33:56.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:33:56.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:33:56.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:33:56.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 20:03:34 UTC on ccycloud.quasar-agiwfo.root.comops.site
23:33:56.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4576543
23:33:56.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:33:56.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:56.223100 11491 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:33:56.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:56.224411 11491 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:33:56.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:56.243289 11499 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:33:57.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:56.244741 11500 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:33:57.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:56.243407 11502 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:33:57.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:57.493506 11501 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
23:33:57.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:57.493680 11491 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
23:33:57.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:57.497821 11491 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:33:57.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:57.500720 11491 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:33:57.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:57.502200 11491 hybrid_clock.cc:648] HybridClock initialized: now 1786318437502150 us; error 68 us; skew 500 ppm
23:33:57.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:57.503103 11491 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:33:57.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 23:33:57 dist-test-slave-7c35 krb5kdc[11195](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786318437, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.235.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
23:33:57.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:57.509415 11491 init.cc:377] Logged in from keytab as kudu/127.10.235.125@KRBTEST.COM (short username kudu)
23:33:57.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:57.515497 11491 webserver.cc:460] Webserver started at http://127.10.235.125:34077/ using document root <none> and password file <none>
23:33:57.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:57.516436 11491 fs_manager.cc:359] Metadata directory not provided
23:33:57.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:57.516660 11491 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:33:57.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:57.524973 11491 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.005s sys 0.000s
23:33:57.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:57.529845 11511 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
23:33:57.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:57.531246 11491 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
23:33:57.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:57.531610 11491 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-1/data,/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-1/wal
23:33:57.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ae4c1992659c4ec78bb1e9066aa24096"
23:33:57.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 23:33:53 on dist-test-slave-7c35"
23:33:57.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:57.534045 11491 fs_report.cc:383] FS layout report
23:33:57.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:33:57.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-1/wal
23:33:57.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-1/wal
23:33:57.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-1/data/data
23:33:57.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:33:57.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:33:57.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:33:57.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:33:57.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:33:57.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:33:57.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:33:57.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:33:57.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:33:57.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:33:57.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:33:57.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:57.574657 11491 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
23:33:57.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:57.580695 11491 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:33:57.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:57.581133 11491 kserver.cc:163] Server-wide thread pool size limit: 3276
23:33:57.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:57.660410 11491 rpc_server.cc:307] RPC server started. Bound to: 127.10.235.125:45571
23:33:57.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:57.660450 11564 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.235.125:45571 every 8 connection(s)
23:33:57.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:57.663446 11491 server_base.cc:1166] Dumped server information to /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-1/data/info.pb
23:33:57.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:57.672195 11181 external_mini_cluster.cc:1388] Started /tmp/dist-test-task5zWf6n/build/tsan/bin/kudu as pid 11491
23:33:57.673 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
23:33:57.673 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.10.235.126:46281
23:33:57.673 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
23:33:57.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:57.675853 11181 external_mini_cluster.cc:1326] Running /tmp/dist-test-task5zWf6n/build/tsan/bin/kudu
23:33:57.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task5zWf6n/build/tsan/bin/kudu
23:33:57.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-0/wal
23:33:57.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-0/data
23:33:57.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:33:57.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:33:57.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:33:57.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:33:57.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:33:57.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:33:57.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-0/logs
23:33:57.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-0/data/info.pb
23:33:57.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:33:57.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:33:57.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:33:57.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:33:57.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:33:57.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:33:57.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:33:57.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:33:57.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
23:33:57.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:33:57.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:33:57.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:33:57.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.235.126:46281
23:33:57.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.235.126
23:33:57.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44905
23:33:57.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/krb5kdc/kudu_127.10.235.126.keytab
23:33:57.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.10.235.126
23:33:57.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
23:33:57.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
23:33:57.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
23:33:57.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.235.84:45015
23:33:57.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:33:57.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:33:57.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:33:57.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:33:57.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.235.126:46281,127.10.235.125:45571,127.10.235.124:34231
23:33:57.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:33:57.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45609/jwks.json
23:33:57.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
23:33:57.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:57.676815 11565 sys_catalog.cc:263] Verifying existing consensus state
23:33:57.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260809 23:33:57.678036 11565 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
23:33:57.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:57.678683 11491 master.cc:537] Master@127.10.235.125:45571 shutting down...
23:33:57.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:57.767750 11491 master.cc:552] Master@127.10.235.125:45571 shutdown complete.
23:33:57.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
23:33:58.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:58.031993 11568 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:33:58.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:58.032573 11568 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:33:58.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:58.033161 11568 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:33:58.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:58.070905 11568 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
23:33:58.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:58.071188 11568 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
23:33:58.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:58.071594 11568 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:33:58.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:58.071820 11568 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
23:33:58.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:58.072114 11568 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
23:33:58.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:58.072436 11568 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
23:33:58.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:58.072671 11568 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45609/jwks.json
23:33:58.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:58.119946 11568 master_runner.cc:386] Master server non-default flags:
23:33:58.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:33:58.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.235.84:45015
23:33:58.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:33:58.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:33:58.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-0/data
23:33:58.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-0/wal
23:33:58.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
23:33:58.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:33:58.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.235.126:46281,127.10.235.125:45571,127.10.235.124:34231
23:33:58.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/krb5kdc/kudu_127.10.235.126.keytab
23:33:58.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.10.235.126
23:33:58.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:33:58.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:33:58.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:33:58.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.235.126:46281
23:33:58.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:33:58.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:33:58.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:33:58.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45609/jwks.json
23:33:58.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
23:33:58.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
23:33:58.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
23:33:58.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:33:58.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:33:58.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-0/data/info.pb
23:33:58.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.235.126
23:33:58.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44905
23:33:58.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:33:58.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:33:58.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:33:58.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:33:58.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:33:58.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-0/logs
23:33:58.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:33:58.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:33:58.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
23:33:58.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
23:33:58.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:33:58.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:33:58.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:33:58.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 20:03:34 UTC on ccycloud.quasar-agiwfo.root.comops.site
23:33:58.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4576543
23:33:58.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:33:58.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:58.121295 11568 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:33:58.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:58.122786 11568 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:33:58.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:58.141247 11577 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:33:59.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:58.141240 11576 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:33:59.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:59.453764 11578 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
23:33:59.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:59.454105 11568 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
23:33:59.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:33:58.141479 11579 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:33:59.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:59.459280 11568 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:33:59.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:59.462690 11568 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:33:59.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:59.464203 11568 hybrid_clock.cc:648] HybridClock initialized: now 1786318439464148 us; error 67 us; skew 500 ppm
23:33:59.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:59.465144 11568 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:33:59.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 23:33:59 dist-test-slave-7c35 krb5kdc[11195](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786318439, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.235.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
23:33:59.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:59.475844 11568 init.cc:377] Logged in from keytab as kudu/127.10.235.126@KRBTEST.COM (short username kudu)
23:33:59.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:59.483021 11568 webserver.cc:460] Webserver started at http://127.10.235.126:44905/ using document root <none> and password file <none>
23:33:59.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:59.484122 11568 fs_manager.cc:359] Metadata directory not provided
23:33:59.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:59.484575 11568 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:33:59.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:59.493229 11568 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.001s sys 0.004s
23:33:59.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:59.498176 11588 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
23:33:59.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:59.499527 11568 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
23:33:59.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:59.499833 11568 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-0/data,/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-0/wal
23:33:59.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ff38ce53b3054dd4a6aa20ba37c685bc"
23:33:59.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 23:33:51 on dist-test-slave-7c35"
23:33:59.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:59.502682 11568 fs_report.cc:383] FS layout report
23:33:59.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:33:59.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-0/wal
23:33:59.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-0/wal
23:33:59.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-0/data/data
23:33:59.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:33:59.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:33:59.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:33:59.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:33:59.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:33:59.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:33:59.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:33:59.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:33:59.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:33:59.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:33:59.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:33:59.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:59.539024 11568 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
23:33:59.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:59.546018 11568 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:33:59.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:59.546687 11568 kserver.cc:163] Server-wide thread pool size limit: 3276
23:33:59.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:59.632546 11568 rpc_server.cc:307] RPC server started. Bound to: 127.10.235.126:46281
23:33:59.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:59.632802 11643 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.235.126:46281 every 8 connection(s)
23:33:59.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:59.636794 11568 server_base.cc:1166] Dumped server information to /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-0/data/info.pb
23:33:59.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:59.642226 11181 external_mini_cluster.cc:1388] Started /tmp/dist-test-task5zWf6n/build/tsan/bin/kudu as pid 11568
23:33:59.643 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
23:33:59.644 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.10.235.124:34231
23:33:59.644 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
23:33:59.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:59.646175 11181 external_mini_cluster.cc:1326] Running /tmp/dist-test-task5zWf6n/build/tsan/bin/kudu
23:33:59.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task5zWf6n/build/tsan/bin/kudu
23:33:59.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-2/wal
23:33:59.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-2/data
23:33:59.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:33:59.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:33:59.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:33:59.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:33:59.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:33:59.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:33:59.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-2/logs
23:33:59.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-2/data/info.pb
23:33:59.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:33:59.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:33:59.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:33:59.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:33:59.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:33:59.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:33:59.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:33:59.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:33:59.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
23:33:59.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:33:59.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:33:59.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:33:59.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.235.124:34231
23:33:59.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.235.124
23:33:59.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45773
23:33:59.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/krb5kdc/kudu_127.10.235.124.keytab
23:33:59.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.10.235.124
23:33:59.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
23:33:59.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
23:33:59.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
23:33:59.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.235.84:45015
23:33:59.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:33:59.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:33:59.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:33:59.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:33:59.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.235.126:46281,127.10.235.125:45571,127.10.235.124:34231
23:33:59.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:33:59.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45609/jwks.json
23:33:59.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
23:33:59.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:59.652050 11644 sys_catalog.cc:263] Verifying existing consensus state
23:33:59.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260809 23:33:59.653194 11644 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
23:33:59.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:59.654529 11568 master.cc:537] Master@127.10.235.126:46281 shutting down...
23:33:59.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:33:59.725340 11568 master.cc:552] Master@127.10.235.126:46281 shutdown complete.
23:33:59.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==================
23:33:59.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: ThreadSanitizer: data race (pid=11568)
23:33:59.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Write of size 1 at 0x7b80000030f8 by main thread:
23:33:59.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 pthread_mutex_destroy /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:1241:3 (kudu+0x443456)
23:33:59.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 kudu::Mutex::~Mutex() /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/mutex.cc:76:12 (libkudu_util.so+0x3d1f90)
23:33:59.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 kudu::CountDownLatch::~CountDownLatch() /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/countdown_latch.h:33:7 (kudu+0x570ee2)
23:33:59.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 kudu::server::ServerBase::~ServerBase() /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/server/server_base.cc:772:1 (libserver_process.so+0x14140f)
23:33:59.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 kudu::kserver::KuduServer::~KuduServer() /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/kserver/kserver.h:41:7 (libmaster.so+0x401baf)
23:33:59.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 kudu::master::Master::~Master() /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/master/master.cc:263:1 (libmaster.so+0x3f7e42)
23:33:59.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 kudu::master::Master::~Master() /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/master/master.cc:261:19 (libmaster.so+0x3f8129)
23:33:59.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 std::__1::default_delete<kudu::master::Master>::operator()(kudu::master::Master*) const /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:2262:5 (libmaster.so+0x435987)
23:33:59.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 std::__1::unique_ptr<kudu::master::Master, std::__1::default_delete<kudu::master::Master> >::reset(kudu::master::Master*) /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:2517:7 (libmaster.so+0x42ea4d)
23:33:59.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 std::__1::unique_ptr<kudu::master::Master, std::__1::default_delete<kudu::master::Master> >::~unique_ptr() /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:2471:19 (libmaster.so+0x42eb2b)
23:33:59.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 kudu::master::RunMasterServer() /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/master/master_runner.cc:477:1 (libmaster.so+0x42beec)
23:33:59.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 kudu::tools::(anonymous namespace)::MasterRun(kudu::tools::RunnerContext const&) /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_action_master.cc:136:10 (kudu+0x58ea59)
23:33:59.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 decltype(std::__1::forward<kudu::Status (*&)(kudu::tools::RunnerContext const&)>(fp)(std::__1::forward<kudu::tools::RunnerContext const&>(fp0))) std::__1::__invoke<kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&>(kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&) /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/type_traits:3899:1 (kudu+0x507ff1)
23:33:59.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 kudu::Status std::__1::__invoke_void_return_wrapper<kudu::Status>::__call<kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&>(kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&) /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/__functional_base:317:16 (kudu+0x507f5c)
23:33:59.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 std::__1::__function::__alloc_func<kudu::Status (*)(kudu::tools::RunnerContext const&), std::__1::allocator<kudu::Status (*)(kudu::tools::RunnerContext const&)>, kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (kudu+0x507ecc)
23:33:59.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 std::__1::__function::__func<kudu::Status (*)(kudu::tools::RunnerContext const&), std::__1::allocator<kudu::Status (*)(kudu::tools::RunnerContext const&)>, kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (kudu+0x506b35)
23:33:59.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 std::__1::__function::__value_func<kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) const /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libkudu_tools_util.so+0x127c5f)
23:33:59.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 std::__1::function<kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) const /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libkudu_tools_util.so+0x11a4c1)
23:33:59.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #18 kudu::tools::Action::Run(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, std::__1::unordered_map<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::hash<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::equal_to<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::allocator<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > > const&, std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) const /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_action.cc:245:10 (libkudu_tools_util.so+0x1181f2)
23:33:59.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #19 kudu::tools::DispatchCommand(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, kudu::tools::Action*, std::__1::deque<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_main.cc:138:15 (kudu+0x61ad20)
23:33:59.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #20 kudu::tools::RunTool(int, char**, bool) /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_main.cc:210:16 (kudu+0x61b657)
23:33:59.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #21 main /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_main.cc:305:10 (kudu+0x61bb9e)
23:33:59.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
23:33:59.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Previous atomic read of size 1 at 0x7b80000030f8 by thread T68:
23:33:59.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 pthread_cond_timedwait /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:1181:3 (kudu+0x452760)
23:33:59.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 kudu::ConditionVariable::WaitUntil(kudu::MonoTime const&) const /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/condition_variable.cc:87:12 (libkudu_util.so+0x2b86da)
23:33:59.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 kudu::CountDownLatch::WaitUntil(kudu::MonoTime const&) const /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/countdown_latch.h:89:18 (kudu+0x573f18)
23:33:59.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 kudu::CountDownLatch::WaitFor(kudu::MonoDelta const&) const /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/countdown_latch.h:99:12 (kudu+0x573d4e)
23:33:59.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 kudu::master::Master::ExpiredReservedTablesDeleterThread() /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/master/master.cc:567:42 (libmaster.so+0x3fa7ce)
23:33:59.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 kudu::master::Master::StartExpiredReservedTablesDeleterThread()::$_2::operator()() const /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/master/master.cc:560:42 (libmaster.so+0x4005c1)
23:33:59.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 decltype(std::__1::forward<kudu::master::Master::StartExpiredReservedTablesDeleterThread()::$_2&>(fp)()) std::__1::__invoke<kudu::master::Master::StartExpiredReservedTablesDeleterThread()::$_2&>(kudu::master::Master::StartExpiredReservedTablesDeleterThread()::$_2&) /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/type_traits:3899:1 (libmaster.so+0x400579)
23:33:59.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::master::Master::StartExpiredReservedTablesDeleterThread()::$_2&>(kudu::master::Master::StartExpiredReservedTablesDeleterThread()::$_2&) /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libmaster.so+0x400509)
23:33:59.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 std::__1::__function::__alloc_func<kudu::master::Master::StartExpiredReservedTablesDeleterThread()::$_2, std::__1::allocator<kudu::master::Master::StartExpiredReservedTablesDeleterThread()::$_2>, void ()>::operator()() /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libmaster.so+0x4004d1)
23:33:59.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 std::__1::__function::__func<kudu::master::Master::StartExpiredReservedTablesDeleterThread()::$_2, std::__1::allocator<kudu::master::Master::StartExpiredReservedTablesDeleterThread()::$_2>, void ()>::operator()() /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libmaster.so+0x3ff7cd)
23:33:59.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 std::__1::__function::__value_func<void ()>::operator()() const /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (kudu+0x580784)
23:33:59.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 std::__1::function<void ()>::operator()() const /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (kudu+0x5805b9)
23:33:59.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 kudu::Thread::SuperviseThread(void*) /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:694:3 (libkudu_util.so+0x43ddc6)
23:33:59.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
23:33:59.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Location is heap block of size 3696 at 0x7b8000003000 allocated by main thread:
23:33:59.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 operator new(unsigned long) /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cpp:64:3 (kudu+0x4c5d0e)
23:33:59.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 kudu::master::RunMasterServer() /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/master/master_runner.cc:394:29 (libmaster.so+0x42b621)
23:33:59.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 kudu::tools::(anonymous namespace)::MasterRun(kudu::tools::RunnerContext const&) /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_action_master.cc:136:10 (kudu+0x58ea59)
23:33:59.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 decltype(std::__1::forward<kudu::Status (*&)(kudu::tools::RunnerContext const&)>(fp)(std::__1::forward<kudu::tools::RunnerContext const&>(fp0))) std::__1::__invoke<kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&>(kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&) /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/type_traits:3899:1 (kudu+0x507ff1)
23:33:59.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 kudu::Status std::__1::__invoke_void_return_wrapper<kudu::Status>::__call<kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&>(kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&) /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/__functional_base:317:16 (kudu+0x507f5c)
23:33:59.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 std::__1::__function::__alloc_func<kudu::Status (*)(kudu::tools::RunnerContext const&), std::__1::allocator<kudu::Status (*)(kudu::tools::RunnerContext const&)>, kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (kudu+0x507ecc)
23:33:59.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 std::__1::__function::__func<kudu::Status (*)(kudu::tools::RunnerContext const&), std::__1::allocator<kudu::Status (*)(kudu::tools::RunnerContext const&)>, kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (kudu+0x506b35)
23:33:59.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 std::__1::__function::__value_func<kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) const /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libkudu_tools_util.so+0x127c5f)
23:33:59.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 std::__1::function<kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) const /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libkudu_tools_util.so+0x11a4c1)
23:33:59.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 kudu::tools::Action::Run(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, std::__1::unordered_map<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::hash<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::equal_to<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::allocator<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > > const&, std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) const /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_action.cc:245:10 (libkudu_tools_util.so+0x1181f2)
23:33:59.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 kudu::tools::DispatchCommand(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, kudu::tools::Action*, std::__1::deque<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_main.cc:138:15 (kudu+0x61ad20)
23:33:59.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 kudu::tools::RunTool(int, char**, bool) /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_main.cc:210:16 (kudu+0x61b657)
23:33:59.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 main /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_main.cc:305:10 (kudu+0x61bb9e)
23:33:59.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
23:33:59.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Thread T68 'expired-reserve' (tid=11646, finished) created by main thread at:
23:33:59.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 pthread_create /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966:3 (kudu+0x426352)
23:33:59.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*) /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:638:15 (libkudu_util.so+0x43d62a)
23:33:59.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 kudu::Thread::Create(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, scoped_refptr<kudu::Thread>*) /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.h:146:12 (kudu+0x570829)
23:33:59.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 kudu::master::Master::StartExpiredReservedTablesDeleterThread() /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/master/master.cc:558:10 (libmaster.so+0x3f9a67)
23:33:59.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 kudu::master::Master::StartAsync() /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/master/master.cc:381:5 (libmaster.so+0x3f8e70)
23:33:59.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 kudu::master::Master::Start() /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/master/master.cc:333:3 (libmaster.so+0x3f8955)
23:33:59.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 kudu::master::RunMasterServer() /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/master/master_runner.cc:396:3 (libmaster.so+0x42b6a9)
23:33:59.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 kudu::tools::(anonymous namespace)::MasterRun(kudu::tools::RunnerContext const&) /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_action_master.cc:136:10 (kudu+0x58ea59)
23:33:59.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 decltype(std::__1::forward<kudu::Status (*&)(kudu::tools::RunnerContext const&)>(fp)(std::__1::forward<kudu::tools::RunnerContext const&>(fp0))) std::__1::__invoke<kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&>(kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&) /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/type_traits:3899:1 (kudu+0x507ff1)
23:33:59.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 kudu::Status std::__1::__invoke_void_return_wrapper<kudu::Status>::__call<kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&>(kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&) /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/__functional_base:317:16 (kudu+0x507f5c)
23:33:59.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 std::__1::__function::__alloc_func<kudu::Status (*)(kudu::tools::RunnerContext const&), std::__1::allocator<kudu::Status (*)(kudu::tools::RunnerContext const&)>, kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (kudu+0x507ecc)
23:33:59.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 std::__1::__function::__func<kudu::Status (*)(kudu::tools::RunnerContext const&), std::__1::allocator<kudu::Status (*)(kudu::tools::RunnerContext const&)>, kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (kudu+0x506b35)
23:33:59.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 std::__1::__function::__value_func<kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) const /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libkudu_tools_util.so+0x127c5f)
23:33:59.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 std::__1::function<kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) const /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libkudu_tools_util.so+0x11a4c1)
23:33:59.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 kudu::tools::Action::Run(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, std::__1::unordered_map<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::hash<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::equal_to<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::allocator<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > > const&, std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) const /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_action.cc:245:10 (libkudu_tools_util.so+0x1181f2)
23:33:59.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 kudu::tools::DispatchCommand(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, kudu::tools::Action*, std::__1::deque<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_main.cc:138:15 (kudu+0x61ad20)
23:33:59.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 kudu::tools::RunTool(int, char**, bool) /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_main.cc:210:16 (kudu+0x61b657)
23:33:59.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 main /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_main.cc:305:10 (kudu+0x61bb9e)
23:33:59.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
23:33:59.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) SUMMARY: ThreadSanitizer: data race /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/mutex.cc:76:12 in kudu::Mutex::~Mutex()
23:33:59.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==================
23:33:59.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
23:34:00.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:00.004843 11647 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:34:00.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:00.005579 11647 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:34:00.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:00.006341 11647 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:34:00.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:00.039642 11647 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
23:34:00.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:00.039958 11647 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
23:34:00.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:00.040416 11647 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:34:00.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:00.040948 11647 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
23:34:00.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:00.041286 11647 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
23:34:00.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:00.041615 11647 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
23:34:00.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:00.041893 11647 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45609/jwks.json
23:34:00.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:00.082398 11647 master_runner.cc:386] Master server non-default flags:
23:34:00.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:34:00.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.235.84:45015
23:34:00.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:34:00.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:34:00.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-2/data
23:34:00.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-2/wal
23:34:00.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
23:34:00.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:34:00.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.235.126:46281,127.10.235.125:45571,127.10.235.124:34231
23:34:00.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/krb5kdc/kudu_127.10.235.124.keytab
23:34:00.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.10.235.124
23:34:00.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:34:00.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:34:00.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:34:00.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.235.124:34231
23:34:00.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:34:00.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:34:00.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:34:00.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45609/jwks.json
23:34:00.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
23:34:00.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
23:34:00.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
23:34:00.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:34:00.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:34:00.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-2/data/info.pb
23:34:00.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.235.124
23:34:00.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45773
23:34:00.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:34:00.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:34:00.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:34:00.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:34:00.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:34:00.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-2/logs
23:34:00.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:34:00.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:34:00.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
23:34:00.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
23:34:00.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:34:00.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:34:00.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:34:00.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 20:03:34 UTC on ccycloud.quasar-agiwfo.root.comops.site
23:34:00.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4576543
23:34:00.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:34:00.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:00.084143 11647 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:34:00.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:00.085651 11647 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:34:00.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:00.107993 11654 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:34:00.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ThreadSanitizer: reported 1 warnings
23:34:01.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:00.112510 11656 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:34:01.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:01.456899 11655 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1351 milliseconds
23:34:01.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:01.456957 11647 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
23:34:01.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:00.109231 11653 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:34:01.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:01.462428 11647 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:34:01.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:01.465806 11647 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:34:01.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:01.467332 11647 hybrid_clock.cc:648] HybridClock initialized: now 1786318441467297 us; error 52 us; skew 500 ppm
23:34:01.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:01.468142 11647 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:34:01.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 23:34:01 dist-test-slave-7c35 krb5kdc[11195](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786318441, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.235.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
23:34:01.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:01.475085 11647 init.cc:377] Logged in from keytab as kudu/127.10.235.124@KRBTEST.COM (short username kudu)
23:34:01.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:01.480898 11647 webserver.cc:460] Webserver started at http://127.10.235.124:45773/ using document root <none> and password file <none>
23:34:01.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:01.481889 11647 fs_manager.cc:359] Metadata directory not provided
23:34:01.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:01.482159 11647 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:34:01.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:01.490471 11647 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.003s sys 0.003s
23:34:01.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:01.495201 11665 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
23:34:01.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:01.496534 11647 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
23:34:01.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:01.496840 11647 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-2/data,/tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-2/wal
23:34:01.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1c0deac92b4c497e92c36e0e1f2b75ac"
23:34:01.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 23:33:55 on dist-test-slave-7c35"
23:34:01.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:01.499640 11647 fs_report.cc:383] FS layout report
23:34:01.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:34:01.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-2/wal
23:34:01.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-2/wal
23:34:01.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-2/data/data
23:34:01.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:34:01.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:34:01.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:34:01.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:34:01.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:34:01.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:34:01.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:34:01.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:34:01.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:34:01.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:34:01.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:34:01.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:01.565562 11647 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
23:34:01.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:01.574744 11647 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:34:01.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:01.575865 11647 kserver.cc:163] Server-wide thread pool size limit: 3276
23:34:01.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:01.661043 11647 rpc_server.cc:307] RPC server started. Bound to: 127.10.235.124:34231
23:34:01.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:01.661164 11722 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.235.124:34231 every 8 connection(s)
23:34:01.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:01.664014 11647 server_base.cc:1166] Dumped server information to /tmp/dist-test-task5zWf6n/test-tmp/mini-kudu-cluster3912383555184816406/master-2/data/info.pb
23:34:01.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:01.669672 11181 external_mini_cluster.cc:1388] Started /tmp/dist-test-task5zWf6n/build/tsan/bin/kudu as pid 11647
23:34:01.670 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
23:34:01.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:01.677520 11723 sys_catalog.cc:263] Verifying existing consensus state
23:34:01.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:01.686547 11723 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac: Bootstrap starting.
23:34:01.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:01.733837 11723 log.cc:824] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac: Log is configured to *not* fsync() on all Append() calls
23:34:01.737 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.10.235.125:45571(127.10.235.125:45571)] connecting to peer
23:34:01.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:01.739779 11723 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
23:34:01.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:01.740860 11723 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac: Bootstrap complete.
23:34:01.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:01.778439 11723 raft_consensus.cc:348] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ff38ce53b3054dd4a6aa20ba37c685bc" member_type: VOTER last_known_addr { host: "127.10.235.126" port: 46281 } } peers { permanent_uuid: "ae4c1992659c4ec78bb1e9066aa24096" member_type: VOTER last_known_addr { host: "127.10.235.125" port: 45571 } } peers { permanent_uuid: "1c0deac92b4c497e92c36e0e1f2b75ac" member_type: VOTER last_known_addr { host: "127.10.235.124" port: 34231 } }
23:34:01.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:01.782068 11723 raft_consensus.cc:374] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
23:34:01.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:01.782404 11723 raft_consensus.cc:729] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1c0deac92b4c497e92c36e0e1f2b75ac, State: Initialized, Role: FOLLOWER
23:34:01.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:01.783530 11723 consensus_queue.cc:260] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ff38ce53b3054dd4a6aa20ba37c685bc" member_type: VOTER last_known_addr { host: "127.10.235.126" port: 46281 } } peers { permanent_uuid: "ae4c1992659c4ec78bb1e9066aa24096" member_type: VOTER last_known_addr { host: "127.10.235.125" port: 45571 } } peers { permanent_uuid: "1c0deac92b4c497e92c36e0e1f2b75ac" member_type: VOTER last_known_addr { host: "127.10.235.124" port: 34231 } }
23:34:01.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:01.786661 11729 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ff38ce53b3054dd4a6aa20ba37c685bc" member_type: VOTER last_known_addr { host: "127.10.235.126" port: 46281 } } peers { permanent_uuid: "ae4c1992659c4ec78bb1e9066aa24096" member_type: VOTER last_known_addr { host: "127.10.235.125" port: 45571 } } peers { permanent_uuid: "1c0deac92b4c497e92c36e0e1f2b75ac" member_type: VOTER last_known_addr { host: "127.10.235.124" port: 34231 } } }
23:34:01.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:01.787771 11729 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [sys.catalog]: This master's current role is: FOLLOWER
23:34:01.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:01.788525 11723 sys_catalog.cc:564] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [sys.catalog]: configured and running, proceeding with master startup.
23:34:01.819 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.10.235.126:46281(127.10.235.126:46281)] connecting to peer
23:34:01.823 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.10.235.124:34231(127.10.235.124:34231)] connecting to peer
23:34:01.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:01.833602 11672 tablet.cc:2369] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac: Can't schedule compaction. Clean time has not been advanced past its initial value.
23:34:01.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:01.835513 11741 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:34:01.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:01.835865 11741 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:34:01.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:01.825013 11647 master_runner.cc:186] Error getting master registration for 127.10.235.126:46281: Network error: Client connection negotiation failed: client connection to 127.10.235.126:46281: connect: Connection refused (error 111),
23:34:01.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:01.858662 11647 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
23:34:01.888 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:01.889 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.235.126:46281(127.10.235.126:46281)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:01.892 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.235.126:46281: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:01.893 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:01.893 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.235.125:45571(127.10.235.125:45571)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:01.895 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.235.125:45571: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:01.896 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.10.235.124:34231(127.10.235.124:34231)] Successfully connected to peer
23:34:02.005 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
23:34:02.016 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.10.235.124
23:34:02.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:02.042961 11729 raft_consensus.cc:482] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
23:34:02.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:02.043598 11729 raft_consensus.cc:504] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ff38ce53b3054dd4a6aa20ba37c685bc" member_type: VOTER last_known_addr { host: "127.10.235.126" port: 46281 } } peers { permanent_uuid: "ae4c1992659c4ec78bb1e9066aa24096" member_type: VOTER last_known_addr { host: "127.10.235.125" port: 45571 } } peers { permanent_uuid: "1c0deac92b4c497e92c36e0e1f2b75ac" member_type: VOTER last_known_addr { host: "127.10.235.124" port: 34231 } }
23:34:02.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:02.048851 11670 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.235.126:46281: connect: Connection refused (error 111)
23:34:02.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:02.049106 11729 leader_election.cc:288] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ff38ce53b3054dd4a6aa20ba37c685bc (127.10.235.126:46281), ae4c1992659c4ec78bb1e9066aa24096 (127.10.235.125:45571)
23:34:02.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:02.056078 11670 leader_election.cc:334] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ff38ce53b3054dd4a6aa20ba37c685bc (127.10.235.126:46281): Network error: Client connection negotiation failed: client connection to 127.10.235.126:46281: connect: Connection refused (error 111)
23:34:02.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:02.057863 11668 leader_election.cc:334] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ae4c1992659c4ec78bb1e9066aa24096 (127.10.235.125:45571): Network error: Client connection negotiation failed: client connection to 127.10.235.125:45571: connect: Connection refused (error 111)
23:34:02.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:02.059142 11668 leader_election.cc:302] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1c0deac92b4c497e92c36e0e1f2b75ac; no voters: ae4c1992659c4ec78bb1e9066aa24096, ff38ce53b3054dd4a6aa20ba37c685bc
23:34:02.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:02.060364 11729 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
23:34:02.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 23:34:02 dist-test-slave-7c35 krb5kdc[11195](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786318435, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.10.235.124@KRBTEST.COM
23:34:02.336 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x8265535e, L:/127.0.0.1:52272 - R:/127.10.235.124:34231] using SASL/GSSAPI
23:34:02.358 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231; will retry
23:34:02.378 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.235.125:45571(127.10.235.125:45571)] connecting to peer
23:34:02.380 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.235.126:46281(127.10.235.126:46281)] connecting to peer
23:34:02.385 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:02.386 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.235.125:45571(127.10.235.125:45571)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:02.386 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.235.125:45571: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:02.392 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:02.392 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.235.126:46281(127.10.235.126:46281)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:02.393 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.235.126:46281: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:02.394 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231; will retry
23:34:02.416 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.235.125:45571(127.10.235.125:45571)] connecting to peer
23:34:02.418 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.235.126:46281(127.10.235.126:46281)] connecting to peer
23:34:02.422 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:02.423 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.235.125:45571(127.10.235.125:45571)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:02.425 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.235.125:45571: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:02.428 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:02.428 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.235.126:46281(127.10.235.126:46281)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:02.428 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.235.126:46281: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:02.431 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231; will retry
23:34:02.437 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.235.125:45571(127.10.235.125:45571)] connecting to peer
23:34:02.439 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.235.126:46281(127.10.235.126:46281)] connecting to peer
23:34:02.440 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:02.440 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.235.125:45571(127.10.235.125:45571)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:02.441 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.235.125:45571: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:02.443 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:02.443 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.235.126:46281(127.10.235.126:46281)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:02.445 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.235.126:46281: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:02.447 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231; will retry
23:34:02.456 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.235.125:45571(127.10.235.125:45571)] connecting to peer
23:34:02.457 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.235.126:46281(127.10.235.126:46281)] connecting to peer
23:34:02.459 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:02.460 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.235.125:45571(127.10.235.125:45571)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:02.462 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.235.125:45571: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:02.464 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:02.464 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.235.126:46281(127.10.235.126:46281)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:02.465 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.235.126:46281: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:02.465 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231; will retry
23:34:02.497 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.235.125:45571(127.10.235.125:45571)] connecting to peer
23:34:02.498 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.235.126:46281(127.10.235.126:46281)] connecting to peer
23:34:02.500 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:02.500 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.235.125:45571(127.10.235.125:45571)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:02.501 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.235.125:45571: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:02.507 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:02.507 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.235.126:46281(127.10.235.126:46281)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:02.507 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.235.126:46281: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:02.509 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231; will retry
23:34:02.537 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.235.125:45571(127.10.235.125:45571)] connecting to peer
23:34:02.538 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.235.126:46281(127.10.235.126:46281)] connecting to peer
23:34:02.540 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:02.540 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.235.125:45571(127.10.235.125:45571)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:02.541 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.235.125:45571: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:02.543 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:02.543 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.235.126:46281(127.10.235.126:46281)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:02.543 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.235.126:46281: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:02.546 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231; will retry
23:34:02.617 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.235.125:45571(127.10.235.125:45571)] connecting to peer
23:34:02.619 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.235.126:46281(127.10.235.126:46281)] connecting to peer
23:34:02.620 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:02.620 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.235.125:45571(127.10.235.125:45571)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:02.621 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.235.125:45571: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:02.624 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:02.624 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.235.126:46281(127.10.235.126:46281)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:02.625 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.235.126:46281: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:02.627 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231; will retry
23:34:02.757 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.235.125:45571(127.10.235.125:45571)] connecting to peer
23:34:02.759 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.235.126:46281(127.10.235.126:46281)] connecting to peer
23:34:02.760 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:02.760 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.235.125:45571(127.10.235.125:45571)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:02.761 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.235.125:45571: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:02.765 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:02.765 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.235.126:46281(127.10.235.126:46281)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:02.766 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.235.126:46281: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:02.768 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231; will retry
23:34:02.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:02.839315 11741 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:34:02.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:02.839636 11741 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:34:02.977 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.235.125:45571(127.10.235.125:45571)] connecting to peer
23:34:02.979 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.235.126:46281(127.10.235.126:46281)] connecting to peer
23:34:02.979 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:02.980 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.235.125:45571(127.10.235.125:45571)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:02.980 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.235.125:45571: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:02.984 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:02.984 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.235.126:46281(127.10.235.126:46281)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:02.985 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.235.126:46281: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:02.987 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231; will retry
23:34:03.317 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.235.125:45571(127.10.235.125:45571)] connecting to peer
23:34:03.319 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.235.126:46281(127.10.235.126:46281)] connecting to peer
23:34:03.319 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:03.319 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.235.125:45571(127.10.235.125:45571)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:03.321 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.235.125:45571: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:03.323 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:03.323 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.235.126:46281(127.10.235.126:46281)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:03.323 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.235.126:46281: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:03.329 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231; will retry
23:34:03.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:03.842646 11741 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:34:03.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:03.843016 11741 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:34:04.017 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.235.125:45571(127.10.235.125:45571)] connecting to peer
23:34:04.018 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.235.126:46281(127.10.235.126:46281)] connecting to peer
23:34:04.019 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:04.019 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.235.125:45571(127.10.235.125:45571)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:04.020 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.235.125:45571: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:04.022 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:04.022 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.235.126:46281(127.10.235.126:46281)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:04.023 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.235.126:46281: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:04.025 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231; will retry
23:34:04.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:04.260578 11752 raft_consensus.cc:482] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
23:34:04.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:04.260954 11752 raft_consensus.cc:504] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ff38ce53b3054dd4a6aa20ba37c685bc" member_type: VOTER last_known_addr { host: "127.10.235.126" port: 46281 } } peers { permanent_uuid: "ae4c1992659c4ec78bb1e9066aa24096" member_type: VOTER last_known_addr { host: "127.10.235.125" port: 45571 } } peers { permanent_uuid: "1c0deac92b4c497e92c36e0e1f2b75ac" member_type: VOTER last_known_addr { host: "127.10.235.124" port: 34231 } }
23:34:04.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:04.262816 11752 leader_election.cc:288] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ff38ce53b3054dd4a6aa20ba37c685bc (127.10.235.126:46281), ae4c1992659c4ec78bb1e9066aa24096 (127.10.235.125:45571)
23:34:04.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:04.270337 11670 leader_election.cc:334] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ff38ce53b3054dd4a6aa20ba37c685bc (127.10.235.126:46281): Network error: Client connection negotiation failed: client connection to 127.10.235.126:46281: connect: Connection refused (error 111)
23:34:04.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:04.270814 11668 leader_election.cc:334] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ae4c1992659c4ec78bb1e9066aa24096 (127.10.235.125:45571): Network error: Client connection negotiation failed: client connection to 127.10.235.125:45571: connect: Connection refused (error 111)
23:34:04.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:04.271345 11668 leader_election.cc:302] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1c0deac92b4c497e92c36e0e1f2b75ac; no voters: ae4c1992659c4ec78bb1e9066aa24096, ff38ce53b3054dd4a6aa20ba37c685bc
23:34:04.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:04.272045 11752 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
23:34:04.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:04.846236 11741 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:34:04.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:04.846537 11741 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:34:05.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:05.585166 11719 debug-util.cc:398] Leaking SignalData structure 0x7b0800073060 after lost signal to thread 11648
23:34:05.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:05.585953 11719 debug-util.cc:398] Leaking SignalData structure 0x7b0800099480 after lost signal to thread 11722
23:34:05.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:05.849449 11741 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:34:05.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:05.849741 11741 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:34:06.477 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.235.125:45571(127.10.235.125:45571)] connecting to peer
23:34:06.479 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.235.126:46281(127.10.235.126:46281)] connecting to peer
23:34:06.480 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:06.480 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.235.125:45571(127.10.235.125:45571)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:06.481 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.235.125:45571: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:06.482 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:06.483 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.235.126:46281(127.10.235.126:46281)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:06.483 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.235.126:46281: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:06.486 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231; will retry
23:34:06.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:06.853137 11741 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:34:06.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:06.853518 11741 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:34:06.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:06.907505 11647 master_runner.cc:186] Error getting master registration for 127.10.235.126:46281: Network error: Client connection negotiation failed: client connection to 127.10.235.126:46281: connect: Connection refused (error 111),
23:34:06.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:06.908001 11647 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
23:34:07.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:07.354497 11759 raft_consensus.cc:482] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
23:34:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:07.354957 11759 raft_consensus.cc:504] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ff38ce53b3054dd4a6aa20ba37c685bc" member_type: VOTER last_known_addr { host: "127.10.235.126" port: 46281 } } peers { permanent_uuid: "ae4c1992659c4ec78bb1e9066aa24096" member_type: VOTER last_known_addr { host: "127.10.235.125" port: 45571 } } peers { permanent_uuid: "1c0deac92b4c497e92c36e0e1f2b75ac" member_type: VOTER last_known_addr { host: "127.10.235.124" port: 34231 } }
23:34:07.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:07.356637 11759 leader_election.cc:288] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ff38ce53b3054dd4a6aa20ba37c685bc (127.10.235.126:46281), ae4c1992659c4ec78bb1e9066aa24096 (127.10.235.125:45571)
23:34:07.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:07.358584 11670 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.235.126:46281: connect: Connection refused (error 111) [suppressed 3 similar messages]
23:34:07.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:07.362558 11670 leader_election.cc:334] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ff38ce53b3054dd4a6aa20ba37c685bc (127.10.235.126:46281): Network error: Client connection negotiation failed: client connection to 127.10.235.126:46281: connect: Connection refused (error 111)
23:34:07.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:07.363225 11668 leader_election.cc:334] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ae4c1992659c4ec78bb1e9066aa24096 (127.10.235.125:45571): Network error: Client connection negotiation failed: client connection to 127.10.235.125:45571: connect: Connection refused (error 111)
23:34:07.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:07.363610 11668 leader_election.cc:302] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1c0deac92b4c497e92c36e0e1f2b75ac; no voters: ae4c1992659c4ec78bb1e9066aa24096, ff38ce53b3054dd4a6aa20ba37c685bc
23:34:07.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:07.364321 11759 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
23:34:07.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:07.856748 11741 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:34:07.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:07.857079 11741 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:34:08.236 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.235.125:45571(127.10.235.125:45571)] connecting to peer
23:34:08.237 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.235.126:46281(127.10.235.126:46281)] connecting to peer
23:34:08.239 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:08.239 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.235.125:45571(127.10.235.125:45571)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:08.239 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.235.125:45571: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:08.240 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:08.241 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.235.126:46281(127.10.235.126:46281)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:08.241 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.235.126:46281: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:08.243 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231; will retry
23:34:08.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:08.860155 11741 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:34:08.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:08.860497 11741 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:34:09.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:09.863427 11741 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:34:09.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:09.863696 11741 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:34:09.917 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.235.125:45571(127.10.235.125:45571)] connecting to peer
23:34:09.918 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.235.126:46281(127.10.235.126:46281)] connecting to peer
23:34:09.920 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:09.920 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.235.125:45571(127.10.235.125:45571)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:09.921 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.235.125:45571: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:09.922 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:09.922 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.235.126:46281(127.10.235.126:46281)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:09.922 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.235.126:46281: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:09.924 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231; will retry
23:34:10.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:10.109800 11765 raft_consensus.cc:482] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
23:34:10.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:10.110731 11765 raft_consensus.cc:504] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ff38ce53b3054dd4a6aa20ba37c685bc" member_type: VOTER last_known_addr { host: "127.10.235.126" port: 46281 } } peers { permanent_uuid: "ae4c1992659c4ec78bb1e9066aa24096" member_type: VOTER last_known_addr { host: "127.10.235.125" port: 45571 } } peers { permanent_uuid: "1c0deac92b4c497e92c36e0e1f2b75ac" member_type: VOTER last_known_addr { host: "127.10.235.124" port: 34231 } }
23:34:10.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:10.113435 11765 leader_election.cc:288] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ff38ce53b3054dd4a6aa20ba37c685bc (127.10.235.126:46281), ae4c1992659c4ec78bb1e9066aa24096 (127.10.235.125:45571)
23:34:10.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:10.120559 11668 leader_election.cc:334] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ae4c1992659c4ec78bb1e9066aa24096 (127.10.235.125:45571): Network error: Client connection negotiation failed: client connection to 127.10.235.125:45571: connect: Connection refused (error 111)
23:34:10.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:10.121270 11670 leader_election.cc:334] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ff38ce53b3054dd4a6aa20ba37c685bc (127.10.235.126:46281): Network error: Client connection negotiation failed: client connection to 127.10.235.126:46281: connect: Connection refused (error 111)
23:34:10.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:10.121608 11670 leader_election.cc:302] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1c0deac92b4c497e92c36e0e1f2b75ac; no voters: ae4c1992659c4ec78bb1e9066aa24096, ff38ce53b3054dd4a6aa20ba37c685bc
23:34:10.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:10.122337 11765 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
23:34:10.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:10.868973 11741 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:34:10.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:10.869287 11741 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:34:11.317 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.235.125:45571(127.10.235.125:45571)] connecting to peer
23:34:11.318 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.235.126:46281(127.10.235.126:46281)] connecting to peer
23:34:11.319 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:11.319 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.235.125:45571(127.10.235.125:45571)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:11.319 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.235.125:45571: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:11.322 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:11.323 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.235.126:46281(127.10.235.126:46281)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:11.323 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.235.126:46281: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:11.325 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231; will retry
23:34:11.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 23:34:11 dist-test-slave-7c35 krb5kdc[11195](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786318451, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.235.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
23:34:11.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:11.479651 11661 init.cc:295] Successfully reacquired a new kerberos TGT
23:34:11.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:11.872377 11741 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:34:11.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:11.872674 11741 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:34:11.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:11.910887 11647 master_runner.cc:186] Error getting master registration for 127.10.235.126:46281: Network error: Client connection negotiation failed: client connection to 127.10.235.126:46281: connect: Connection refused (error 111),
23:34:11.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:11.911365 11647 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
23:34:12.497 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.235.125:45571(127.10.235.125:45571)] connecting to peer
23:34:12.499 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.235.126:46281(127.10.235.126:46281)] connecting to peer
23:34:12.500 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:12.500 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.235.125:45571(127.10.235.125:45571)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:12.501 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.235.125:45571: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:12.502 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:12.502 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.235.126:46281(127.10.235.126:46281)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:12.502 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.235.126:46281: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:12.505 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231; will retry
23:34:12.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:12.875639 11741 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:34:12.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:12.875922 11741 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:34:13.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:13.879084 11741 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:34:13.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:13.879359 11741 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:34:14.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:14.010267 11775 raft_consensus.cc:482] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
23:34:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:14.010701 11775 raft_consensus.cc:504] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ff38ce53b3054dd4a6aa20ba37c685bc" member_type: VOTER last_known_addr { host: "127.10.235.126" port: 46281 } } peers { permanent_uuid: "ae4c1992659c4ec78bb1e9066aa24096" member_type: VOTER last_known_addr { host: "127.10.235.125" port: 45571 } } peers { permanent_uuid: "1c0deac92b4c497e92c36e0e1f2b75ac" member_type: VOTER last_known_addr { host: "127.10.235.124" port: 34231 } }
23:34:14.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:14.012547 11775 leader_election.cc:288] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ff38ce53b3054dd4a6aa20ba37c685bc (127.10.235.126:46281), ae4c1992659c4ec78bb1e9066aa24096 (127.10.235.125:45571)
23:34:14.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:14.014647 11668 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.235.125:45571: connect: Connection refused (error 111) [suppressed 3 similar messages]
23:34:14.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:14.017475 11670 leader_election.cc:334] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ff38ce53b3054dd4a6aa20ba37c685bc (127.10.235.126:46281): Network error: Client connection negotiation failed: client connection to 127.10.235.126:46281: connect: Connection refused (error 111)
23:34:14.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:14.017970 11668 leader_election.cc:334] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ae4c1992659c4ec78bb1e9066aa24096 (127.10.235.125:45571): Network error: Client connection negotiation failed: client connection to 127.10.235.125:45571: connect: Connection refused (error 111)
23:34:14.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:14.018375 11668 leader_election.cc:302] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1c0deac92b4c497e92c36e0e1f2b75ac; no voters: ae4c1992659c4ec78bb1e9066aa24096, ff38ce53b3054dd4a6aa20ba37c685bc
23:34:14.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:14.019171 11775 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
23:34:14.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:14.882203 11741 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:34:14.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:14.882481 11741 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:34:15.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:15.885422 11741 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:34:15.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:15.885691 11741 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:34:16.157 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.235.125:45571(127.10.235.125:45571)] connecting to peer
23:34:16.158 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.235.126:46281(127.10.235.126:46281)] connecting to peer
23:34:16.159 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:16.159 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.235.125:45571(127.10.235.125:45571)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:16.160 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.235.125:45571: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:16.162 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:16.163 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.235.126:46281(127.10.235.126:46281)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:16.163 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.235.126:46281: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:16.164 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231; will retry
23:34:16.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:16.888706 11741 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:34:16.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:16.889047 11741 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:34:16.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:16.914018 11647 master_runner.cc:186] Error getting master registration for 127.10.235.126:46281: Network error: Client connection negotiation failed: client connection to 127.10.235.126:46281: connect: Connection refused (error 111),
23:34:16.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:16.914471 11647 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
23:34:17.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:17.230213 11782 raft_consensus.cc:482] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
23:34:17.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:17.230611 11782 raft_consensus.cc:504] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ff38ce53b3054dd4a6aa20ba37c685bc" member_type: VOTER last_known_addr { host: "127.10.235.126" port: 46281 } } peers { permanent_uuid: "ae4c1992659c4ec78bb1e9066aa24096" member_type: VOTER last_known_addr { host: "127.10.235.125" port: 45571 } } peers { permanent_uuid: "1c0deac92b4c497e92c36e0e1f2b75ac" member_type: VOTER last_known_addr { host: "127.10.235.124" port: 34231 } }
23:34:17.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:17.232191 11782 leader_election.cc:288] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ff38ce53b3054dd4a6aa20ba37c685bc (127.10.235.126:46281), ae4c1992659c4ec78bb1e9066aa24096 (127.10.235.125:45571)
23:34:17.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:17.235934 11670 leader_election.cc:334] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ff38ce53b3054dd4a6aa20ba37c685bc (127.10.235.126:46281): Network error: Client connection negotiation failed: client connection to 127.10.235.126:46281: connect: Connection refused (error 111)
23:34:17.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:17.236354 11668 leader_election.cc:334] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ae4c1992659c4ec78bb1e9066aa24096 (127.10.235.125:45571): Network error: Client connection negotiation failed: client connection to 127.10.235.125:45571: connect: Connection refused (error 111)
23:34:17.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:17.236658 11668 leader_election.cc:302] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1c0deac92b4c497e92c36e0e1f2b75ac; no voters: ae4c1992659c4ec78bb1e9066aa24096, ff38ce53b3054dd4a6aa20ba37c685bc
23:34:17.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:17.237263 11782 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
23:34:17.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:17.892204 11741 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:34:17.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:17.892514 11741 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:34:18.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:18.895440 11741 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:34:18.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:18.895762 11741 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:34:19.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:19.898793 11741 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:34:19.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:19.899112 11741 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:34:20.157 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.235.125:45571(127.10.235.125:45571)] connecting to peer
23:34:20.158 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.235.126:46281(127.10.235.126:46281)] connecting to peer
23:34:20.159 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:20.159 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.235.125:45571(127.10.235.125:45571)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:20.160 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.235.125:45571: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:20.161 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:20.161 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.235.126:46281(127.10.235.126:46281)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:20.161 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.235.126:46281: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:20.164 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231; will retry
23:34:20.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:20.902623 11741 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:34:20.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:20.903040 11741 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:34:21.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:21.360790 11790 raft_consensus.cc:482] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
23:34:21.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:21.361446 11790 raft_consensus.cc:504] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ff38ce53b3054dd4a6aa20ba37c685bc" member_type: VOTER last_known_addr { host: "127.10.235.126" port: 46281 } } peers { permanent_uuid: "ae4c1992659c4ec78bb1e9066aa24096" member_type: VOTER last_known_addr { host: "127.10.235.125" port: 45571 } } peers { permanent_uuid: "1c0deac92b4c497e92c36e0e1f2b75ac" member_type: VOTER last_known_addr { host: "127.10.235.124" port: 34231 } }
23:34:21.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:21.363504 11790 leader_election.cc:288] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ff38ce53b3054dd4a6aa20ba37c685bc (127.10.235.126:46281), ae4c1992659c4ec78bb1e9066aa24096 (127.10.235.125:45571)
23:34:21.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:21.366170 11668 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.235.125:45571: connect: Connection refused (error 111)
23:34:21.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:21.366163 11670 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.235.126:46281: connect: Connection refused (error 111) [suppressed 3 similar messages]
23:34:21.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:21.369238 11668 leader_election.cc:334] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ae4c1992659c4ec78bb1e9066aa24096 (127.10.235.125:45571): Network error: Client connection negotiation failed: client connection to 127.10.235.125:45571: connect: Connection refused (error 111)
23:34:21.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:21.369999 11670 leader_election.cc:334] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ff38ce53b3054dd4a6aa20ba37c685bc (127.10.235.126:46281): Network error: Client connection negotiation failed: client connection to 127.10.235.126:46281: connect: Connection refused (error 111)
23:34:21.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:21.370401 11670 leader_election.cc:302] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1c0deac92b4c497e92c36e0e1f2b75ac; no voters: ae4c1992659c4ec78bb1e9066aa24096, ff38ce53b3054dd4a6aa20ba37c685bc
23:34:21.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:21.371189 11790 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
23:34:21.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 23:34:21 dist-test-slave-7c35 krb5kdc[11195](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786318461, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.235.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
23:34:21.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:21.482045 11661 init.cc:295] Successfully reacquired a new kerberos TGT
23:34:21.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:21.905969 11741 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:34:21.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:21.906277 11741 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:34:21.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:21.917548 11647 master_runner.cc:186] Error getting master registration for 127.10.235.126:46281: Network error: Client connection negotiation failed: client connection to 127.10.235.126:46281: connect: Connection refused (error 111),
23:34:21.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:21.918102 11647 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
23:34:22.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:22.909113 11741 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:34:22.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:22.909400 11741 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:34:23.340 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.235.125:45571(127.10.235.125:45571)] connecting to peer
23:34:23.342 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.235.126:46281(127.10.235.126:46281)] connecting to peer
23:34:23.343 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:23.343 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.235.125:45571(127.10.235.125:45571)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:23.343 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.235.125:45571: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:23.344 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:23.344 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.235.126:46281(127.10.235.126:46281)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:23.344 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.235.126:46281: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:23.346 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231; will retry
23:34:23.797 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.235.125:45571(127.10.235.125:45571)] connecting to peer
23:34:23.798 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.235.126:46281(127.10.235.126:46281)] connecting to peer
23:34:23.798 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:23.798 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.235.125:45571(127.10.235.125:45571)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:23.799 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.235.125:45571: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:23.800 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:23.800 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.235.126:46281(127.10.235.126:46281)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:23.800 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.235.126:46281: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:23.803 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231; will retry
23:34:23.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:23.864820 11799 raft_consensus.cc:482] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
23:34:23.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:23.865435 11799 raft_consensus.cc:504] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ff38ce53b3054dd4a6aa20ba37c685bc" member_type: VOTER last_known_addr { host: "127.10.235.126" port: 46281 } } peers { permanent_uuid: "ae4c1992659c4ec78bb1e9066aa24096" member_type: VOTER last_known_addr { host: "127.10.235.125" port: 45571 } } peers { permanent_uuid: "1c0deac92b4c497e92c36e0e1f2b75ac" member_type: VOTER last_known_addr { host: "127.10.235.124" port: 34231 } }
23:34:23.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:23.867384 11799 leader_election.cc:288] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ff38ce53b3054dd4a6aa20ba37c685bc (127.10.235.126:46281), ae4c1992659c4ec78bb1e9066aa24096 (127.10.235.125:45571)
23:34:23.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:23.872059 11670 leader_election.cc:334] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ff38ce53b3054dd4a6aa20ba37c685bc (127.10.235.126:46281): Network error: Client connection negotiation failed: client connection to 127.10.235.126:46281: connect: Connection refused (error 111)
23:34:23.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:23.872771 11668 leader_election.cc:334] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ae4c1992659c4ec78bb1e9066aa24096 (127.10.235.125:45571): Network error: Client connection negotiation failed: client connection to 127.10.235.125:45571: connect: Connection refused (error 111)
23:34:23.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:23.873142 11668 leader_election.cc:302] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1c0deac92b4c497e92c36e0e1f2b75ac; no voters: ae4c1992659c4ec78bb1e9066aa24096, ff38ce53b3054dd4a6aa20ba37c685bc
23:34:23.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:23.873706 11799 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
23:34:23.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:23.912416 11741 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:34:23.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:23.912717 11741 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:34:24.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:24.915670 11741 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:34:24.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:24.915946 11741 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:34:25.637 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.235.125:45571(127.10.235.125:45571)] connecting to peer
23:34:25.638 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.235.126:46281(127.10.235.126:46281)] connecting to peer
23:34:25.639 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:25.639 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.235.125:45571(127.10.235.125:45571)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:25.639 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.235.125:45571: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:25.640 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:25.640 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.235.126:46281(127.10.235.126:46281)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:25.640 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.235.126:46281: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:25.643 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231; will retry
23:34:25.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:25.919214 11741 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:34:25.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:25.919480 11741 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:34:26.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:26.920876 11647 master_runner.cc:186] Error getting master registration for 127.10.235.126:46281: Network error: Client connection negotiation failed: client connection to 127.10.235.126:46281: connect: Connection refused (error 111),
23:34:26.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:26.921334 11647 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
23:34:26.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:26.922957 11741 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:34:26.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:26.923259 11741 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:34:27.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:27.926084 11741 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:34:27.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:27.926374 11741 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:34:28.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:28.929546 11741 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:34:28.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:28.929836 11741 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:34:29.276 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.235.125:45571(127.10.235.125:45571)] connecting to peer
23:34:29.277 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.235.126:46281(127.10.235.126:46281)] connecting to peer
23:34:29.278 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:29.278 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.235.125:45571(127.10.235.125:45571)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:29.278 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.235.125:45571: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:29.279 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:29.279 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.235.126:46281(127.10.235.126:46281)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:29.279 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.235.126:46281: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:29.281 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231; will retry
23:34:29.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:29.932981 11741 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1c0deac92b4c497e92c36e0e1f2b75ac: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:34:29.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 23:34:29.933277 11741 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:34:29.996 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.235.125:45571(127.10.235.125:45571)] connecting to peer
23:34:29.997 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.235.126:46281(127.10.235.126:46281)] connecting to peer
23:34:29.998 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:29.998 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.235.125:45571(127.10.235.125:45571)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:29.998 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.235.125:45571: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571
23:34:29.999 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:30.000 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.235.126:46281(127.10.235.126:46281)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:30.000 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.235.126:46281: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
23:34:30.003 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231; will retry
23:34:30.007 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=28329), Traces: [0ms] refreshing cache from master, [44ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.125:45571, [138ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.126:46281, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.124:34231, [209ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.126:46281: Network error: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [212ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.125:45571: Network error: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571, [673ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.124:34231: OK, [678ms] delaying RPC due to: Service unavailable: Master config (127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571, [696ms] refreshing cache from master, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.125:45571, [699ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.126:46281, [699ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.124:34231, [705ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.125:45571: Network error: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571, [710ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.124:34231: OK, [712ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.126:46281: Network error: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [713ms] delaying RPC due to: Service unavailable: Master config (127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [735ms] refreshing cache from master, [735ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.125:45571, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.126:46281, [741ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.124:34231, [742ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.125:45571: Network error: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571, [747ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.126:46281: Network error: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [749ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.124:34231: OK, [750ms] delaying RPC due to: Service unavailable: Master config (127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [756ms] refreshing cache from master, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.125:45571, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.126:46281, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.124:34231, [759ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.125:45571: Network error: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571, [762ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.126:46281: Network error: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [765ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.124:34231: OK, [766ms] delaying RPC due to: Service unavailable: Master config (127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [775ms] refreshing cache from master, [775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.125:45571, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.126:46281, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.124:34231, [779ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.125:45571: Network error: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571, [782ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.124:34231: OK, [783ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.126:46281: Network error: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [785ms] delaying RPC due to: Service unavailable: Master config (127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [815ms] refreshing cache from master, [816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.125:45571, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.126:46281, [818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.124:34231, [819ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.125:45571: Network error: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571, [826ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.126:46281: Network error: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [828ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.124:34231: OK, [828ms] delaying RPC due to: Service unavailable: Master config (127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [855ms] refreshing cache from master, [855ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.125:45571, [857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.126:46281, [858ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.124:34231, [859ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.125:45571: Network error: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571, [862ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.126:46281: Network error: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [864ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.124:34231: OK, [865ms] delaying RPC due to: Service unavailable: Master config (127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [935ms] refreshing cache from master, [936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.125:45571, [938ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.126:46281, [939ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.125:45571: Network error: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571, [939ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.124:34231, [943ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.126:46281: Network error: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [945ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.124:34231: OK, [947ms] delaying RPC due to: Service unavailable: Master config (127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [1076ms] refreshing cache from master, [1076ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.125:45571, [1077ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.126:46281, [1079ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.125:45571: Network error: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571, [1081ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.124:34231, [1084ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.126:46281: Network error: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [1086ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.124:34231: OK, [1087ms] delaying RPC due to: Service unavailable: Master config (127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [1295ms] refreshing cache from master, [1296ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.125:45571, [1297ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.126:46281, [1299ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.125:45571: Network error: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571, [1299ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.124:34231, [1303ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.126:46281: Network error: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [1305ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.124:34231: OK, [1306ms] delaying RPC due to: Service unavailable: Master config (127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [1635ms] refreshing cache from master, [1636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.125:45571, [1637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.126:46281, [1638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.124:34231, [1640ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.125:45571: Network error: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571, [1642ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.126:46281: Network error: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [1647ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.124:34231: OK, [1648ms] delaying RPC due to: Service unavailable: Master config (127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [2335ms] refreshing cache from master, [2336ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.125:45571, [2337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.126:46281, [2338ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.124:34231, [2339ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.125:45571: Network error: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571, [2342ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.126:46281: Network error: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [2343ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.124:34231: OK, [2344ms] delaying RPC due to: Service unavailable: Master config (127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [4796ms] refreshing cache from master, [4796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.125:45571, [4797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.126:46281, [4799ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.124:34231, [4800ms] too many traces: truncated at 100 traces, deferred=Deferred@360121393(state=PENDING, result=null, callback=(continuation of Deferred@1164840193 after retry RPC after error@1044584914) -> (continuation of Deferred@644786361 after retry RPC after error@185956041) -> (continuation of Deferred@1042707631 after retry RPC after error@403535664) -> (continuation of Deferred@1265971176 after retry RPC after error@1802939249) -> (continuation of Deferred@1727539644 after retry RPC after error@364865188) -> (continuation of Deferred@2010583444 after retry RPC after error@690225468) -> (continuation of Deferred@238450797 after retry RPC after error@1624417609) -> (continuation of Deferred@801526646 after retry RPC after error@652380755) -> (continuation of Deferred@455769074 after retry RPC after error@1856470323) -> (continuation of Deferred@1936186994 after retry RPC after error@1097167455) -> (continuation of Deferred@156791914 after retry RPC after error@1556340375) -> (continuation of Deferred@286924595 after retry RPC after error@2064085189) -> (continuation of Deferred@825663723 after retry RPC after error@1942218556) -> (continuation of Deferred@845254128 after retry RPC after error@1923109409) -> (continuation of Deferred@1989514598 after retry RPC after error@436430479) -> (continuation of Deferred@607348274 after retry RPC after error@1416759759) -> (continuation of Deferred@244750466 after retry RPC after error@262185153) -> (continuation of Deferred@807224817 after retry RPC after error@1961168460) -> (continuation of Deferred@1317915354 after retry RPC after error@1151458658) -> (continuation of Deferred@508111562 after retry RPC after error@1107476513) -> (continuation of Deferred@1652099489 after retry RPC after error@679055937) -> (continuation of Deferred@2142364414 after retry RPC after error@736503521) -> (continuation of Deferred@33035837 after retry RPC after error@1702324185), errback=(continuation of Deferred@1164840193 after retry RPC after error@1044584914) -> (continuation of Deferred@644786361 after retry RPC after error@185956041) -> (continuation of Deferred@1042707631 after retry RPC after error@403535664) -> (continuation of Deferred@1265971176 after retry RPC after error@1802939249) -> (continuation of Deferred@1727539644 after retry RPC after error@364865188) -> (continuation of Deferred@2010583444 after retry RPC after error@690225468) -> (continuation of Deferred@238450797 after retry RPC after error@1624417609) -> (continuation of Deferred@801526646 after retry RPC after error@652380755) -> (continuation of Deferred@455769074 after retry RPC after error@1856470323) -> (continuation of Deferred@1936186994 after retry RPC after error@1097167455) -> (continuation of Deferred@156791914 after retry RPC after error@1556340375) -> (continuation of Deferred@286924595 after retry RPC after error@2064085189) -> (continuation of Deferred@825663723 after retry RPC after error@1942218556) -> (continuation of Deferred@845254128 after retry RPC after error@1923109409) -> (continuation of Deferred@1989514598 after retry RPC after error@436430479) -> (continuation of Deferred@607348274 after retry RPC after error@1416759759) -> (continuation of Deferred@244750466 after retry RPC after error@262185153) -> (continuation of Deferred@807224817 after retry RPC after error@1961168460) -> (continuation of Deferred@1317915354 after retry RPC after error@1151458658) -> (continuation of Deferred@508111562 after retry RPC after error@1107476513) -> (continuation of Deferred@1652099489 after retry RPC after error@679055937) -> (continuation of Deferred@2142364414 after retry RPC after error@736503521) -> (continuation of Deferred@33035837 after retry RPC after error@1702324185)))
23:34:30.013 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.10.235.124:34231(127.10.235.124:34231)] disconnecting while in state READY
23:34:30.014 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.10.235.125:45571(127.10.235.125:45571)] disconnecting while in state TERMINATED
23:34:30.014 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.10.235.126:46281(127.10.235.126:46281)] disconnecting while in state TERMINATED
23:34:30.019 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
23:34:30.021 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
23:34:30.021 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.10.235.124:34231(127.10.235.124:34231)] handling channelInactive
23:34:30.021 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.235.124:34231(127.10.235.124:34231)] cleaning up while in state READY due to: connection closed
23:34:30.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:30.028091 11181 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task5zWf6n/build/tsan/bin/kudu with pid 11647
23:34:31.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-09T23:34:31Z chronyd exiting
23:34:32.305 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
23:34:32.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:32.306569 10389 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task5zWf6n/build/tsan/bin/kudu with pid 10700
23:34:32.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:32.351761 10389 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task5zWf6n/build/tsan/bin/kudu with pid 10874
23:34:32.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:32.383700 10389 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task5zWf6n/build/tsan/bin/kudu with pid 11028
23:34:32.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:32.418188 10389 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task5zWf6n/build/tsan/bin/kudu with pid 10404
23:34:32.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:32.458547 10389 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task5zWf6n/build/tsan/bin/kudu with pid 10482
23:34:32.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 23:34:32.498422 10389 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task5zWf6n/build/tsan/bin/kudu with pid 10564
23:34:33.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-09T23:34:33Z chronyd exiting
23:34:34.708 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=28329), Traces: [0ms] refreshing cache from master, [44ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.125:45571, [138ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.126:46281, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.124:34231, [209ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.126:46281: Network error: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [212ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.125:45571: Network error: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571, [673ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.124:34231: OK, [678ms] delaying RPC due to: Service unavailable: Master config (127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571, [696ms] refreshing cache from master, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.125:45571, [699ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.126:46281, [699ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.124:34231, [705ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.125:45571: Network error: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571, [710ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.124:34231: OK, [712ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.126:46281: Network error: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [713ms] delaying RPC due to: Service unavailable: Master config (127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [735ms] refreshing cache from master, [735ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.125:45571, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.126:46281, [741ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.124:34231, [742ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.125:45571: Network error: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571, [747ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.126:46281: Network error: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [749ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.124:34231: OK, [750ms] delaying RPC due to: Service unavailable: Master config (127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [756ms] refreshing cache from master, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.125:45571, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.126:46281, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.124:34231, [759ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.125:45571: Network error: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571, [762ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.126:46281: Network error: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [765ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.124:34231: OK, [766ms] delaying RPC due to: Service unavailable: Master config (127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [775ms] refreshing cache from master, [775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.125:45571, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.126:46281, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.124:34231, [779ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.125:45571: Network error: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571, [782ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.124:34231: OK, [783ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.126:46281: Network error: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [785ms] delaying RPC due to: Service unavailable: Master config (127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [815ms] refreshing cache from master, [816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.125:45571, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.126:46281, [818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.124:34231, [819ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.125:45571: Network error: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571, [826ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.126:46281: Network error: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [828ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.124:34231: OK, [828ms] delaying RPC due to: Service unavailable: Master config (127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [855ms] refreshing cache from master, [855ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.125:45571, [857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.126:46281, [858ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.124:34231, [859ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.125:45571: Network error: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571, [862ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.126:46281: Network error: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [864ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.124:34231: OK, [865ms] delaying RPC due to: Service unavailable: Master config (127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [935ms] refreshing cache from master, [936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.125:45571, [938ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.126:46281, [939ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.125:45571: Network error: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571, [939ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.124:34231, [943ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.126:46281: Network error: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [945ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.124:34231: OK, [947ms] delaying RPC due to: Service unavailable: Master config (127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [1076ms] refreshing cache from master, [1076ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.125:45571, [1077ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.126:46281, [1079ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.125:45571: Network error: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571, [1081ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.124:34231, [1084ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.126:46281: Network error: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [1086ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.124:34231: OK, [1087ms] delaying RPC due to: Service unavailable: Master config (127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [1295ms] refreshing cache from master, [1296ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.125:45571, [1297ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.126:46281, [1299ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.125:45571: Network error: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571, [1299ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.124:34231, [1303ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.126:46281: Network error: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [1305ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.124:34231: OK, [1306ms] delaying RPC due to: Service unavailable: Master config (127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [1635ms] refreshing cache from master, [1636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.125:45571, [1637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.126:46281, [1638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.124:34231, [1640ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.125:45571: Network error: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571, [1642ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.126:46281: Network error: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [1647ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.124:34231: OK, [1648ms] delaying RPC due to: Service unavailable: Master config (127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [2335ms] refreshing cache from master, [2336ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.125:45571, [2337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.126:46281, [2338ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.124:34231, [2339ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.125:45571: Network error: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571, [2342ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.126:46281: Network error: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [2343ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.124:34231: OK, [2344ms] delaying RPC due to: Service unavailable: Master config (127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [4796ms] refreshing cache from master, [4796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.125:45571, [4797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.126:46281, [4799ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.124:34231, [4800ms] too many traces: truncated at 100 traces, deferred=Deferred@360121393(state=PENDING, result=null, callback=(continuation of Deferred@1164840193 after retry RPC after error@1044584914) -> (continuation of Deferred@644786361 after retry RPC after error@185956041) -> (continuation of Deferred@1042707631 after retry RPC after error@403535664) -> (continuation of Deferred@1265971176 after retry RPC after error@1802939249) -> (continuation of Deferred@1727539644 after retry RPC after error@364865188) -> (continuation of Deferred@2010583444 after retry RPC after error@690225468) -> (continuation of Deferred@238450797 after retry RPC after error@1624417609) -> (continuation of Deferred@801526646 after retry RPC after error@652380755) -> (continuation of Deferred@455769074 after retry RPC after error@1856470323) -> (continuation of Deferred@1936186994 after retry RPC after error@1097167455) -> (continuation of Deferred@156791914 after retry RPC after error@1556340375) -> (continuation of Deferred@286924595 after retry RPC after error@2064085189) -> (continuation of Deferred@825663723 after retry RPC after error@1942218556) -> (continuation of Deferred@845254128 after retry RPC after error@1923109409) -> (continuation of Deferred@1989514598 after retry RPC after error@436430479) -> (continuation of Deferred@607348274 after retry RPC after error@1416759759) -> (continuation of Deferred@244750466 after retry RPC after error@262185153) -> (continuation of Deferred@807224817 after retry RPC after error@1961168460) -> (continuation of Deferred@1317915354 after retry RPC after error@1151458658) -> (continuation of Deferred@508111562 after retry RPC after error@1107476513) -> (continuation of Deferred@1652099489 after retry RPC after error@679055937) -> (continuation of Deferred@2142364414 after retry RPC after error@736503521) -> (continuation of Deferred@33035837 after retry RPC after error@1702324185), errback=(continuation of Deferred@1164840193 after retry RPC after error@1044584914) -> (continuation of Deferred@644786361 after retry RPC after error@185956041) -> (continuation of Deferred@1042707631 after retry RPC after error@403535664) -> (continuation of Deferred@1265971176 after retry RPC after error@1802939249) -> (continuation of Deferred@1727539644 after retry RPC after error@364865188) -> (continuation of Deferred@2010583444 after retry RPC after error@690225468) -> (continuation of Deferred@238450797 after retry RPC after error@1624417609) -> (continuation of Deferred@801526646 after retry RPC after error@652380755) -> (continuation of Deferred@455769074 after retry RPC after error@1856470323) -> (continuation of Deferred@1936186994 after retry RPC after error@1097167455) -> (continuation of Deferred@156791914 after retry RPC after error@1556340375) -> (continuation of Deferred@286924595 after retry RPC after error@2064085189) -> (continuation of Deferred@825663723 after retry RPC after error@1942218556) -> (continuation of Deferred@845254128 after retry RPC after error@1923109409) -> (continuation of Deferred@1989514598 after retry RPC after error@436430479) -> (continuation of Deferred@607348274 after retry RPC after error@1416759759) -> (continuation of Deferred@244750466 after retry RPC after error@262185153) -> (continuation of Deferred@807224817 after retry RPC after error@1961168460) -> (continuation of Deferred@1317915354 after retry RPC after error@1151458658) -> (continuation of Deferred@508111562 after retry RPC after error@1107476513) -> (continuation of Deferred@1652099489 after retry RPC after error@679055937) -> (continuation of Deferred@2142364414 after retry RPC after error@736503521) -> (continuation of Deferred@33035837 after retry RPC after error@1702324185)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.access$200(DefaultPromise.java:35) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise$1.run(DefaultPromise.java:503) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.10.235.126:46281
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=28329), Traces: [0ms] refreshing cache from master, [44ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.125:45571, [138ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.126:46281, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.124:34231, [209ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.126:46281: Network error: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [212ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.125:45571: Network error: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571, [673ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.124:34231: OK, [678ms] delaying RPC due to: Service unavailable: Master config (127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571, [696ms] refreshing cache from master, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.125:45571, [699ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.126:46281, [699ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.124:34231, [705ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.125:45571: Network error: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571, [710ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.124:34231: OK, [712ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.126:46281: Network error: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [713ms] delaying RPC due to: Service unavailable: Master config (127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [735ms] refreshing cache from master, [735ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.125:45571, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.126:46281, [741ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.124:34231, [742ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.125:45571: Network error: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571, [747ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.126:46281: Network error: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [749ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.124:34231: OK, [750ms] delaying RPC due to: Service unavailable: Master config (127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [756ms] refreshing cache from master, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.125:45571, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.126:46281, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.124:34231, [759ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.125:45571: Network error: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571, [762ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.126:46281: Network error: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [765ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.124:34231: OK, [766ms] delaying RPC due to: Service unavailable: Master config (127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [775ms] refreshing cache from master, [775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.125:45571, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.126:46281, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.124:34231, [779ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.125:45571: Network error: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571, [782ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.124:34231: OK, [783ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.126:46281: Network error: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [785ms] delaying RPC due to: Service unavailable: Master config (127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [815ms] refreshing cache from master, [816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.125:45571, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.126:46281, [818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.124:34231, [819ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.125:45571: Network error: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571, [826ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.126:46281: Network error: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [828ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.124:34231: OK, [828ms] delaying RPC due to: Service unavailable: Master config (127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [855ms] refreshing cache from master, [855ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.125:45571, [857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.126:46281, [858ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.124:34231, [859ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.125:45571: Network error: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571, [862ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.126:46281: Network error: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [864ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.124:34231: OK, [865ms] delaying RPC due to: Service unavailable: Master config (127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [935ms] refreshing cache from master, [936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.125:45571, [938ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.126:46281, [939ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.125:45571: Network error: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571, [939ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.124:34231, [943ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.126:46281: Network error: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [945ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.124:34231: OK, [947ms] delaying RPC due to: Service unavailable: Master config (127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [1076ms] refreshing cache from master, [1076ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.125:45571, [1077ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.126:46281, [1079ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.125:45571: Network error: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571, [1081ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.124:34231, [1084ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.126:46281: Network error: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [1086ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.124:34231: OK, [1087ms] delaying RPC due to: Service unavailable: Master config (127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [1295ms] refreshing cache from master, [1296ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.125:45571, [1297ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.126:46281, [1299ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.125:45571: Network error: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571, [1299ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.124:34231, [1303ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.126:46281: Network error: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [1305ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.124:34231: OK, [1306ms] delaying RPC due to: Service unavailable: Master config (127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [1635ms] refreshing cache from master, [1636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.125:45571, [1637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.126:46281, [1638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.124:34231, [1640ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.125:45571: Network error: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571, [1642ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.126:46281: Network error: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [1647ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.124:34231: OK, [1648ms] delaying RPC due to: Service unavailable: Master config (127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [2335ms] refreshing cache from master, [2336ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.125:45571, [2337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.126:46281, [2338ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.124:34231, [2339ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.125:45571: Network error: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571, [2342ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.126:46281: Network error: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [2343ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.124:34231: OK, [2344ms] delaying RPC due to: Service unavailable: Master config (127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [4796ms] refreshing cache from master, [4796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.125:45571, [4797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.126:46281, [4799ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.124:34231, [4800ms] too many traces: truncated at 100 traces, deferred=Deferred@360121393(state=PENDING, result=null, callback=(continuation of Deferred@1164840193 after retry RPC after error@1044584914) -> (continuation of Deferred@644786361 after retry RPC after error@185956041) -> (continuation of Deferred@1042707631 after retry RPC after error@403535664) -> (continuation of Deferred@1265971176 after retry RPC after error@1802939249) -> (continuation of Deferred@1727539644 after retry RPC after error@364865188) -> (continuation of Deferred@2010583444 after retry RPC after error@690225468) -> (continuation of Deferred@238450797 after retry RPC after error@1624417609) -> (continuation of Deferred@801526646 after retry RPC after error@652380755) -> (continuation of Deferred@455769074 after retry RPC after error@1856470323) -> (continuation of Deferred@1936186994 after retry RPC after error@1097167455) -> (continuation of Deferred@156791914 after retry RPC after error@1556340375) -> (continuation of Deferred@286924595 after retry RPC after error@2064085189) -> (continuation of Deferred@825663723 after retry RPC after error@1942218556) -> (continuation of Deferred@845254128 after retry RPC after error@1923109409) -> (continuation of Deferred@1989514598 after retry RPC after error@436430479) -> (continuation of Deferred@607348274 after retry RPC after error@1416759759) -> (continuation of Deferred@244750466 after retry RPC after error@262185153) -> (continuation of Deferred@807224817 after retry RPC after error@1961168460) -> (continuation of Deferred@1317915354 after retry RPC after error@1151458658) -> (continuation of Deferred@508111562 after retry RPC after error@1107476513) -> (continuation of Deferred@1652099489 after retry RPC after error@679055937) -> (continuation of Deferred@2142364414 after retry RPC after error@736503521) -> (continuation of Deferred@33035837 after retry RPC after error@1702324185), errback=(continuation of Deferred@1164840193 after retry RPC after error@1044584914) -> (continuation of Deferred@644786361 after retry RPC after error@185956041) -> (continuation of Deferred@1042707631 after retry RPC after error@403535664) -> (continuation of Deferred@1265971176 after retry RPC after error@1802939249) -> (continuation of Deferred@1727539644 after retry RPC after error@364865188) -> (continuation of Deferred@2010583444 after retry RPC after error@690225468) -> (continuation of Deferred@238450797 after retry RPC after error@1624417609) -> (continuation of Deferred@801526646 after retry RPC after error@652380755) -> (continuation of Deferred@455769074 after retry RPC after error@1856470323) -> (continuation of Deferred@1936186994 after retry RPC after error@1097167455) -> (continuation of Deferred@156791914 after retry RPC after error@1556340375) -> (continuation of Deferred@286924595 after retry RPC after error@2064085189) -> (continuation of Deferred@825663723 after retry RPC after error@1942218556) -> (continuation of Deferred@845254128 after retry RPC after error@1923109409) -> (continuation of Deferred@1989514598 after retry RPC after error@436430479) -> (continuation of Deferred@607348274 after retry RPC after error@1416759759) -> (continuation of Deferred@244750466 after retry RPC after error@262185153) -> (continuation of Deferred@807224817 after retry RPC after error@1961168460) -> (continuation of Deferred@1317915354 after retry RPC after error@1151458658) -> (continuation of Deferred@508111562 after retry RPC after error@1107476513) -> (continuation of Deferred@1652099489 after retry RPC after error@679055937) -> (continuation of Deferred@2142364414 after retry RPC after error@736503521) -> (continuation of Deferred@33035837 after retry RPC after error@1702324185)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557)
at io.netty.util.concurrent.DefaultPromise.access$200(DefaultPromise.java:35)
at io.netty.util.concurrent.DefaultPromise$1.run(DefaultPromise.java:503)
at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174)
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167)
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569)
... 5 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.10.235.126:46281
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
23:34:35.138 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
23:34:35.139 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=28329), Traces: [0ms] refreshing cache from master, [44ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.125:45571, [138ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.126:46281, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.124:34231, [209ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.126:46281: Network error: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [212ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.125:45571: Network error: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571, [673ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.124:34231: OK, [678ms] delaying RPC due to: Service unavailable: Master config (127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571, [696ms] refreshing cache from master, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.125:45571, [699ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.126:46281, [699ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.124:34231, [705ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.125:45571: Network error: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571, [710ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.124:34231: OK, [712ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.126:46281: Network error: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [713ms] delaying RPC due to: Service unavailable: Master config (127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [735ms] refreshing cache from master, [735ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.125:45571, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.126:46281, [741ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.124:34231, [742ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.125:45571: Network error: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571, [747ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.126:46281: Network error: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [749ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.124:34231: OK, [750ms] delaying RPC due to: Service unavailable: Master config (127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [756ms] refreshing cache from master, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.125:45571, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.126:46281, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.124:34231, [759ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.125:45571: Network error: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571, [762ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.126:46281: Network error: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [765ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.124:34231: OK, [766ms] delaying RPC due to: Service unavailable: Master config (127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [775ms] refreshing cache from master, [775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.125:45571, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.126:46281, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.124:34231, [779ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.125:45571: Network error: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571, [782ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.124:34231: OK, [783ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.126:46281: Network error: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [785ms] delaying RPC due to: Service unavailable: Master config (127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [815ms] refreshing cache from master, [816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.125:45571, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.126:46281, [818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.124:34231, [819ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.125:45571: Network error: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571, [826ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.126:46281: Network error: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [828ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.124:34231: OK, [828ms] delaying RPC due to: Service unavailable: Master config (127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [855ms] refreshing cache from master, [855ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.125:45571, [857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.126:46281, [858ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.124:34231, [859ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.125:45571: Network error: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571, [862ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.126:46281: Network error: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [864ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.124:34231: OK, [865ms] delaying RPC due to: Service unavailable: Master config (127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [935ms] refreshing cache from master, [936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.125:45571, [938ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.126:46281, [939ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.125:45571: Network error: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571, [939ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.124:34231, [943ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.126:46281: Network error: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [945ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.124:34231: OK, [947ms] delaying RPC due to: Service unavailable: Master config (127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [1076ms] refreshing cache from master, [1076ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.125:45571, [1077ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.126:46281, [1079ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.125:45571: Network error: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571, [1081ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.124:34231, [1084ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.126:46281: Network error: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [1086ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.124:34231: OK, [1087ms] delaying RPC due to: Service unavailable: Master config (127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [1295ms] refreshing cache from master, [1296ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.125:45571, [1297ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.126:46281, [1299ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.125:45571: Network error: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571, [1299ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.124:34231, [1303ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.126:46281: Network error: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [1305ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.124:34231: OK, [1306ms] delaying RPC due to: Service unavailable: Master config (127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [1635ms] refreshing cache from master, [1636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.125:45571, [1637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.126:46281, [1638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.124:34231, [1640ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.125:45571: Network error: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571, [1642ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.126:46281: Network error: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [1647ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.124:34231: OK, [1648ms] delaying RPC due to: Service unavailable: Master config (127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [2335ms] refreshing cache from master, [2336ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.125:45571, [2337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.126:46281, [2338ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.124:34231, [2339ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.125:45571: Network error: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571, [2342ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.126:46281: Network error: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [2343ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.124:34231: OK, [2344ms] delaying RPC due to: Service unavailable: Master config (127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [4796ms] refreshing cache from master, [4796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.125:45571, [4797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.126:46281, [4799ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.124:34231, [4800ms] too many traces: truncated at 100 traces, deferred=Deferred@360121393(state=PENDING, result=null, callback=(continuation of Deferred@1164840193 after retry RPC after error@1044584914) -> (continuation of Deferred@644786361 after retry RPC after error@185956041) -> (continuation of Deferred@1042707631 after retry RPC after error@403535664) -> (continuation of Deferred@1265971176 after retry RPC after error@1802939249) -> (continuation of Deferred@1727539644 after retry RPC after error@364865188) -> (continuation of Deferred@2010583444 after retry RPC after error@690225468) -> (continuation of Deferred@238450797 after retry RPC after error@1624417609) -> (continuation of Deferred@801526646 after retry RPC after error@652380755) -> (continuation of Deferred@455769074 after retry RPC after error@1856470323) -> (continuation of Deferred@1936186994 after retry RPC after error@1097167455) -> (continuation of Deferred@156791914 after retry RPC after error@1556340375) -> (continuation of Deferred@286924595 after retry RPC after error@2064085189) -> (continuation of Deferred@825663723 after retry RPC after error@1942218556) -> (continuation of Deferred@845254128 after retry RPC after error@1923109409) -> (continuation of Deferred@1989514598 after retry RPC after error@436430479) -> (continuation of Deferred@607348274 after retry RPC after error@1416759759) -> (continuation of Deferred@244750466 after retry RPC after error@262185153) -> (continuation of Deferred@807224817 after retry RPC after error@1961168460) -> (continuation of Deferred@1317915354 after retry RPC after error@1151458658) -> (continuation of Deferred@508111562 after retry RPC after error@1107476513) -> (continuation of Deferred@1652099489 after retry RPC after error@679055937) -> (continuation of Deferred@2142364414 after retry RPC after error@736503521) -> (continuation of Deferred@33035837 after retry RPC after error@1702324185), errback=(continuation of Deferred@1164840193 after retry RPC after error@1044584914) -> (continuation of Deferred@644786361 after retry RPC after error@185956041) -> (continuation of Deferred@1042707631 after retry RPC after error@403535664) -> (continuation of Deferred@1265971176 after retry RPC after error@1802939249) -> (continuation of Deferred@1727539644 after retry RPC after error@364865188) -> (continuation of Deferred@2010583444 after retry RPC after error@690225468) -> (continuation of Deferred@238450797 after retry RPC after error@1624417609) -> (continuation of Deferred@801526646 after retry RPC after error@652380755) -> (continuation of Deferred@455769074 after retry RPC after error@1856470323) -> (continuation of Deferred@1936186994 after retry RPC after error@1097167455) -> (continuation of Deferred@156791914 after retry RPC after error@1556340375) -> (continuation of Deferred@286924595 after retry RPC after error@2064085189) -> (continuation of Deferred@825663723 after retry RPC after error@1942218556) -> (continuation of Deferred@845254128 after retry RPC after error@1923109409) -> (continuation of Deferred@1989514598 after retry RPC after error@436430479) -> (continuation of Deferred@607348274 after retry RPC after error@1416759759) -> (continuation of Deferred@244750466 after retry RPC after error@262185153) -> (continuation of Deferred@807224817 after retry RPC after error@1961168460) -> (continuation of Deferred@1317915354 after retry RPC after error@1151458658) -> (continuation of Deferred@508111562 after retry RPC after error@1107476513) -> (continuation of Deferred@1652099489 after retry RPC after error@679055937) -> (continuation of Deferred@2142364414 after retry RPC after error@736503521) -> (continuation of Deferred@33035837 after retry RPC after error@1702324185)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.access$200(DefaultPromise.java:35) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise$1.run(DefaultPromise.java:503) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.10.235.126:46281
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=28329), Traces: [0ms] refreshing cache from master, [44ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.125:45571, [138ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.126:46281, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.124:34231, [209ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.126:46281: Network error: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [212ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.125:45571: Network error: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571, [673ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.124:34231: OK, [678ms] delaying RPC due to: Service unavailable: Master config (127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571, [696ms] refreshing cache from master, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.125:45571, [699ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.126:46281, [699ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.124:34231, [705ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.125:45571: Network error: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571, [710ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.124:34231: OK, [712ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.126:46281: Network error: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [713ms] delaying RPC due to: Service unavailable: Master config (127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [735ms] refreshing cache from master, [735ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.125:45571, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.126:46281, [741ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.124:34231, [742ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.125:45571: Network error: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571, [747ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.126:46281: Network error: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [749ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.124:34231: OK, [750ms] delaying RPC due to: Service unavailable: Master config (127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [756ms] refreshing cache from master, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.125:45571, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.126:46281, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.124:34231, [759ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.125:45571: Network error: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571, [762ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.126:46281: Network error: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [765ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.124:34231: OK, [766ms] delaying RPC due to: Service unavailable: Master config (127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [775ms] refreshing cache from master, [775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.125:45571, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.126:46281, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.124:34231, [779ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.125:45571: Network error: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571, [782ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.124:34231: OK, [783ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.126:46281: Network error: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [785ms] delaying RPC due to: Service unavailable: Master config (127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [815ms] refreshing cache from master, [816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.125:45571, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.126:46281, [818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.124:34231, [819ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.125:45571: Network error: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571, [826ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.126:46281: Network error: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [828ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.124:34231: OK, [828ms] delaying RPC due to: Service unavailable: Master config (127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [855ms] refreshing cache from master, [855ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.125:45571, [857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.126:46281, [858ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.124:34231, [859ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.125:45571: Network error: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571, [862ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.126:46281: Network error: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [864ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.124:34231: OK, [865ms] delaying RPC due to: Service unavailable: Master config (127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [935ms] refreshing cache from master, [936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.125:45571, [938ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.126:46281, [939ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.125:45571: Network error: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571, [939ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.124:34231, [943ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.126:46281: Network error: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [945ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.124:34231: OK, [947ms] delaying RPC due to: Service unavailable: Master config (127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [1076ms] refreshing cache from master, [1076ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.125:45571, [1077ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.126:46281, [1079ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.125:45571: Network error: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571, [1081ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.124:34231, [1084ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.126:46281: Network error: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [1086ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.124:34231: OK, [1087ms] delaying RPC due to: Service unavailable: Master config (127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [1295ms] refreshing cache from master, [1296ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.125:45571, [1297ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.126:46281, [1299ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.125:45571: Network error: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571, [1299ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.124:34231, [1303ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.126:46281: Network error: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [1305ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.124:34231: OK, [1306ms] delaying RPC due to: Service unavailable: Master config (127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [1635ms] refreshing cache from master, [1636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.125:45571, [1637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.126:46281, [1638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.124:34231, [1640ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.125:45571: Network error: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571, [1642ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.126:46281: Network error: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [1647ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.124:34231: OK, [1648ms] delaying RPC due to: Service unavailable: Master config (127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [2335ms] refreshing cache from master, [2336ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.125:45571, [2337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.126:46281, [2338ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.124:34231, [2339ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.125:45571: Network error: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571, [2342ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.126:46281: Network error: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [2343ms] Sub RPC ConnectToMaster: received response from server master-127.10.235.124:34231: OK, [2344ms] delaying RPC due to: Service unavailable: Master config (127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281, [4796ms] refreshing cache from master, [4796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.125:45571, [4797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.126:46281, [4799ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.235.124:34231, [4800ms] too many traces: truncated at 100 traces, deferred=Deferred@360121393(state=PENDING, result=null, callback=(continuation of Deferred@1164840193 after retry RPC after error@1044584914) -> (continuation of Deferred@644786361 after retry RPC after error@185956041) -> (continuation of Deferred@1042707631 after retry RPC after error@403535664) -> (continuation of Deferred@1265971176 after retry RPC after error@1802939249) -> (continuation of Deferred@1727539644 after retry RPC after error@364865188) -> (continuation of Deferred@2010583444 after retry RPC after error@690225468) -> (continuation of Deferred@238450797 after retry RPC after error@1624417609) -> (continuation of Deferred@801526646 after retry RPC after error@652380755) -> (continuation of Deferred@455769074 after retry RPC after error@1856470323) -> (continuation of Deferred@1936186994 after retry RPC after error@1097167455) -> (continuation of Deferred@156791914 after retry RPC after error@1556340375) -> (continuation of Deferred@286924595 after retry RPC after error@2064085189) -> (continuation of Deferred@825663723 after retry RPC after error@1942218556) -> (continuation of Deferred@845254128 after retry RPC after error@1923109409) -> (continuation of Deferred@1989514598 after retry RPC after error@436430479) -> (continuation of Deferred@607348274 after retry RPC after error@1416759759) -> (continuation of Deferred@244750466 after retry RPC after error@262185153) -> (continuation of Deferred@807224817 after retry RPC after error@1961168460) -> (continuation of Deferred@1317915354 after retry RPC after error@1151458658) -> (continuation of Deferred@508111562 after retry RPC after error@1107476513) -> (continuation of Deferred@1652099489 after retry RPC after error@679055937) -> (continuation of Deferred@2142364414 after retry RPC after error@736503521) -> (continuation of Deferred@33035837 after retry RPC after error@1702324185), errback=(continuation of Deferred@1164840193 after retry RPC after error@1044584914) -> (continuation of Deferred@644786361 after retry RPC after error@185956041) -> (continuation of Deferred@1042707631 after retry RPC after error@403535664) -> (continuation of Deferred@1265971176 after retry RPC after error@1802939249) -> (continuation of Deferred@1727539644 after retry RPC after error@364865188) -> (continuation of Deferred@2010583444 after retry RPC after error@690225468) -> (continuation of Deferred@238450797 after retry RPC after error@1624417609) -> (continuation of Deferred@801526646 after retry RPC after error@652380755) -> (continuation of Deferred@455769074 after retry RPC after error@1856470323) -> (continuation of Deferred@1936186994 after retry RPC after error@1097167455) -> (continuation of Deferred@156791914 after retry RPC after error@1556340375) -> (continuation of Deferred@286924595 after retry RPC after error@2064085189) -> (continuation of Deferred@825663723 after retry RPC after error@1942218556) -> (continuation of Deferred@845254128 after retry RPC after error@1923109409) -> (continuation of Deferred@1989514598 after retry RPC after error@436430479) -> (continuation of Deferred@607348274 after retry RPC after error@1416759759) -> (continuation of Deferred@244750466 after retry RPC after error@262185153) -> (continuation of Deferred@807224817 after retry RPC after error@1961168460) -> (continuation of Deferred@1317915354 after retry RPC after error@1151458658) -> (continuation of Deferred@508111562 after retry RPC after error@1107476513) -> (continuation of Deferred@1652099489 after retry RPC after error@679055937) -> (continuation of Deferred@2142364414 after retry RPC after error@736503521) -> (continuation of Deferred@33035837 after retry RPC after error@1702324185)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.10.235.125:45571,127.10.235.126:46281,127.10.235.124:34231) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.125:45571(127.10.235.125:45571): Connection refused: /127.10.235.125:45571,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.235.126:46281(127.10.235.126:46281): Connection refused: /127.10.235.126:46281
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557)
at io.netty.util.concurrent.DefaultPromise.access$200(DefaultPromise.java:35)
at io.netty.util.concurrent.DefaultPromise$1.run(DefaultPromise.java:503)
at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174)
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167)
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569)
... 5 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.10.235.126:46281
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)