11:53:37.289 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
11:53:37.302 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskrtJWFm/test-tmp
11:53:37.305 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
11:53:37.309 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskrtJWFm/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
11:53:37.536 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
11:53:37.706 [DEBUG - main] (MiniKuduCluster.java:195) Response:
11:53:37.707 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
11:53:37.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:37.711077 3355 mini_oidc.cc:152] Starting JWKS server
11:53:37.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:37.715682 3355 webserver.cc:460] Webserver started at http://127.0.0.1:35053/ using document root <none> and password file <none>
11:53:37.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:37.716318 3355 mini_oidc.cc:189] Starting OIDC Discovery server
11:53:37.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:37.718557 3355 webserver.cc:460] Webserver started at http://127.0.0.1:33671/ using document root <none> and password file <none>
11:53:37.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T11:53:37Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
11:53:37.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T11:53:37Z Disabled control of system clock
11:53:37.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:37.774035 3355 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskrtJWFm/build/tsan/bin/kudu
11:53:37.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrtJWFm/build/tsan/bin/kudu
11:53:37.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/master-0/wal
11:53:37.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/master-0/data
11:53:37.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:53:37.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:53:37.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:53:37.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:53:37.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:53:37.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:53:37.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/master-0/logs
11:53:37.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/master-0/data/info.pb
11:53:37.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:53:37.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:53:37.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:53:37.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:53:37.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:53:37.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:53:37.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:53:37.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:53:37.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
11:53:37.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:53:37.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:53:37.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:53:37.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.70.254:44597
11:53:37.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.70.254
11:53:37.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:53:37.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.70.212:39605
11:53:37.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:53:37.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:53:37.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:53:37.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:53:37.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.70.254:44597,127.3.70.253:36585,127.3.70.252:34019
11:53:37.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:53:37.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35053/jwks.json with env {}
11:53:38.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:38.093554 3367 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:53:38.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:38.094125 3367 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:53:38.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:38.094656 3367 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:53:38.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:38.133562 3367 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
11:53:38.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:38.133992 3367 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:53:38.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:38.134320 3367 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
11:53:38.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:38.134655 3367 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
11:53:38.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:38.135007 3367 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
11:53:38.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:38.135293 3367 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35053/jwks.json
11:53:38.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:38.170162 3367 master_runner.cc:386] Master server non-default flags:
11:53:38.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:53:38.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.70.212:39605
11:53:38.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:53:38.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:53:38.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/master-0/data
11:53:38.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/master-0/wal
11:53:38.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:53:38.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.70.254:44597,127.3.70.253:36585,127.3.70.252:34019
11:53:38.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:53:38.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:53:38.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:53:38.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.70.254:44597
11:53:38.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:53:38.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:53:38.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:53:38.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35053/jwks.json
11:53:38.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:53:38.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:53:38.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/master-0/data/info.pb
11:53:38.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.70.254
11:53:38.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:53:38.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:53:38.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:53:38.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:53:38.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:53:38.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:53:38.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/master-0/logs
11:53:38.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:53:38.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:53:38.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:53:38.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
11:53:38.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:53:38.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:53:38.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:53:38.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 10:52:50 UTC on ccycloud.quasar-peqjre.root.comops.site
11:53:38.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679900
11:53:38.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:53:38.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:38.171746 3367 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:53:38.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:38.174131 3367 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:53:38.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:38.197173 3376 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:53:39.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:38.197243 3374 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:53:39.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:38.197425 3373 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:53:39.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:38.198256 3367 server_base.cc:1034] running on GCE node
11:53:39.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:39.423735 3367 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:53:39.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:39.427539 3367 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:53:39.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:39.429143 3367 hybrid_clock.cc:648] HybridClock initialized: now 1790682819429035 us; error 72 us; skew 500 ppm
11:53:39.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:39.429991 3367 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:53:39.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:39.445539 3367 webserver.cc:460] Webserver started at http://127.3.70.254:34711/ using document root <none> and password file <none>
11:53:39.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:39.446619 3367 fs_manager.cc:359] Metadata directory not provided
11:53:39.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:39.446837 3367 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:53:39.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:39.447292 3367 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
11:53:39.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:39.452832 3367 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/master-0/data/instance:
11:53:39.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "73746926f8964f6590c9b9277e05f3bc"
11:53:39.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:53:39 on dist-test-slave-cvvz"
11:53:39.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:39.454107 3367 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/master-0/wal/instance:
11:53:39.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "73746926f8964f6590c9b9277e05f3bc"
11:53:39.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:53:39 on dist-test-slave-cvvz"
11:53:39.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:39.462082 3367 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
11:53:39.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:39.467962 3383 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:53:39.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:39.469249 3367 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
11:53:39.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:39.469626 3367 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/master-0/data,/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/master-0/wal
11:53:39.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "73746926f8964f6590c9b9277e05f3bc"
11:53:39.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:53:39 on dist-test-slave-cvvz"
11:53:39.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:39.469942 3367 fs_report.cc:383] FS layout report
11:53:39.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:53:39.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/master-0/wal
11:53:39.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/master-0/wal
11:53:39.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/master-0/data/data
11:53:39.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:53:39.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:53:39.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:53:39.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:53:39.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:53:39.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:53:39.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:53:39.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:53:39.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:53:39.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:53:39.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:53:39.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:39.507484 3367 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:53:39.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:39.508934 3367 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:53:39.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:39.509460 3367 kserver.cc:163] Server-wide thread pool size limit: 3276
11:53:39.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:39.585544 3367 rpc_server.cc:307] RPC server started. Bound to: 127.3.70.254:44597
11:53:39.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:39.585650 3434 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.3.70.254:44597 every 8 connection(s)
11:53:39.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:39.588337 3367 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/master-0/data/info.pb
11:53:39.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:39.594686 3435 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
11:53:39.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:39.595350 3355 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskrtJWFm/build/tsan/bin/kudu as pid 3367
11:53:39.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:39.595724 3355 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/master-0/wal/instance
11:53:39.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:39.599014 3355 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskrtJWFm/build/tsan/bin/kudu
11:53:39.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrtJWFm/build/tsan/bin/kudu
11:53:39.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/master-1/wal
11:53:39.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/master-1/data
11:53:39.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:53:39.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:53:39.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:53:39.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:53:39.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:53:39.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:53:39.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/master-1/logs
11:53:39.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/master-1/data/info.pb
11:53:39.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:53:39.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:53:39.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:53:39.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:53:39.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:53:39.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:53:39.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:53:39.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:53:39.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
11:53:39.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:53:39.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:53:39.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:53:39.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.70.253:36585
11:53:39.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.70.253
11:53:39.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:53:39.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.70.212:39605
11:53:39.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:53:39.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:53:39.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:53:39.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:53:39.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.70.254:44597,127.3.70.253:36585,127.3.70.252:34019
11:53:39.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:53:39.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35053/jwks.json with env {}
11:53:39.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:39.611846 3435 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.70.254" port: 44597 } has no permanent_uuid. Determining permanent_uuid...
11:53:39.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:39.645673 3435 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.70.253" port: 36585 } has no permanent_uuid. Determining permanent_uuid...
11:53:39.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:39.648088 3385 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.70.253:36585: connect: Connection refused (error 111)
11:53:39.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:39.652802 3435 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.70.253:36585: Network error: Client connection negotiation failed: client connection to 127.3.70.253:36585: connect: Connection refused (error 111)
11:53:39.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:39.702452 3435 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.70.253" port: 36585 } attempt: 1
11:53:39.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:39.706401 3435 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.70.253:36585: Network error: Client connection negotiation failed: client connection to 127.3.70.253:36585: connect: Connection refused (error 111)
11:53:39.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:39.774878 3435 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.70.253" port: 36585 } attempt: 2
11:53:39.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:39.778595 3435 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.70.253:36585: Network error: Client connection negotiation failed: client connection to 127.3.70.253:36585: connect: Connection refused (error 111)
11:53:39.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:39.870155 3435 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.70.253" port: 36585 } attempt: 3
11:53:39.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:39.874559 3435 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.70.253:36585: Network error: Client connection negotiation failed: client connection to 127.3.70.253:36585: connect: Connection refused (error 111)
11:53:39.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:39.916047 3437 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:53:39.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:39.916762 3437 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:53:39.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:39.917511 3437 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:53:39.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:39.948392 3437 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
11:53:39.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:39.948791 3437 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:53:39.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:39.949122 3437 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
11:53:39.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:39.949455 3437 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
11:53:39.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:39.949797 3437 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
11:53:39.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:39.950075 3437 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35053/jwks.json
11:53:39.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:39.984751 3437 master_runner.cc:386] Master server non-default flags:
11:53:39.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:53:39.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.70.212:39605
11:53:39.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:53:39.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:53:39.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/master-1/data
11:53:39.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/master-1/wal
11:53:39.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:53:39.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.70.254:44597,127.3.70.253:36585,127.3.70.252:34019
11:53:39.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:53:39.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:53:39.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:53:39.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.70.253:36585
11:53:39.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:53:39.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:53:39.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:53:39.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35053/jwks.json
11:53:39.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:53:39.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:53:39.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/master-1/data/info.pb
11:53:39.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.70.253
11:53:39.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:53:39.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:53:39.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:53:39.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:53:39.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:53:39.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:53:39.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/master-1/logs
11:53:39.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:53:39.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:53:39.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:53:39.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
11:53:39.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:53:39.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:53:39.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:53:39.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 10:52:50 UTC on ccycloud.quasar-peqjre.root.comops.site
11:53:39.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679900
11:53:39.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:53:39.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:39.986356 3437 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:53:39.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:39.988655 3437 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:53:40.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:40.007875 3447 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:53:40.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:40.017973 3435 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.70.253" port: 36585 } attempt: 4
11:53:40.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:40.022491 3435 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.70.253:36585: Network error: Client connection negotiation failed: client connection to 127.3.70.253:36585: connect: Connection refused (error 111)
11:53:40.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:40.321934 3435 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.70.253" port: 36585 } attempt: 5
11:53:40.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:40.326085 3435 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.70.253:36585: Network error: Client connection negotiation failed: client connection to 127.3.70.253:36585: connect: Connection refused (error 111)
11:53:40.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:40.873775 3435 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.70.253" port: 36585 } attempt: 6
11:53:40.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:40.886379 3435 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.70.253:36585: Network error: Client connection negotiation failed: client connection to 127.3.70.253:36585: connect: Connection refused (error 111)
11:53:41.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:40.008203 3446 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:53:41.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:40.008284 3449 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:53:41.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:40.009563 3437 server_base.cc:1034] running on GCE node
11:53:41.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:41.306631 3437 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:53:41.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:41.309844 3437 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:53:41.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:41.311332 3437 hybrid_clock.cc:648] HybridClock initialized: now 1790682821311277 us; error 66 us; skew 500 ppm
11:53:41.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:41.312412 3437 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:53:41.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:41.318989 3437 webserver.cc:460] Webserver started at http://127.3.70.253:33061/ using document root <none> and password file <none>
11:53:41.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:41.319995 3437 fs_manager.cc:359] Metadata directory not provided
11:53:41.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:41.320178 3437 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:53:41.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:41.320554 3437 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
11:53:41.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:41.325742 3437 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/master-1/data/instance:
11:53:41.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5356a27f5bb44efaacb57e10ccc592ec"
11:53:41.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:53:41 on dist-test-slave-cvvz"
11:53:41.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:41.327108 3437 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/master-1/wal/instance:
11:53:41.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5356a27f5bb44efaacb57e10ccc592ec"
11:53:41.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:53:41 on dist-test-slave-cvvz"
11:53:41.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:41.336138 3437 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.005s sys 0.004s
11:53:41.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:41.342128 3458 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:53:41.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:41.343289 3437 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
11:53:41.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:41.343657 3437 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/master-1/data,/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/master-1/wal
11:53:41.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5356a27f5bb44efaacb57e10ccc592ec"
11:53:41.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:53:41 on dist-test-slave-cvvz"
11:53:41.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:41.344067 3437 fs_report.cc:383] FS layout report
11:53:41.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:53:41.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/master-1/wal
11:53:41.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/master-1/wal
11:53:41.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/master-1/data/data
11:53:41.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:53:41.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:53:41.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:53:41.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:53:41.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:53:41.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:53:41.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:53:41.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:53:41.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:53:41.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:53:41.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:53:41.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:41.384217 3437 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:53:41.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:41.386040 3437 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:53:41.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:41.386592 3437 kserver.cc:163] Server-wide thread pool size limit: 3276
11:53:41.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:41.466143 3437 rpc_server.cc:307] RPC server started. Bound to: 127.3.70.253:36585
11:53:41.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:41.466240 3509 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.3.70.253:36585 every 8 connection(s)
11:53:41.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:41.469247 3437 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/master-1/data/info.pb
11:53:41.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:41.472465 3355 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskrtJWFm/build/tsan/bin/kudu as pid 3437
11:53:41.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:41.472986 3355 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/master-1/wal/instance
11:53:41.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:41.476531 3510 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
11:53:41.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:41.476557 3355 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskrtJWFm/build/tsan/bin/kudu
11:53:41.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrtJWFm/build/tsan/bin/kudu
11:53:41.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/master-2/wal
11:53:41.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/master-2/data
11:53:41.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:53:41.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:53:41.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:53:41.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:53:41.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:53:41.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:53:41.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/master-2/logs
11:53:41.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/master-2/data/info.pb
11:53:41.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:53:41.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:53:41.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:53:41.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:53:41.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:53:41.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:53:41.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:53:41.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:53:41.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
11:53:41.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:53:41.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:53:41.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:53:41.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.70.252:34019
11:53:41.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.70.252
11:53:41.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:53:41.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.70.212:39605
11:53:41.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:53:41.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:53:41.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:53:41.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:53:41.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.70.254:44597,127.3.70.253:36585,127.3.70.252:34019
11:53:41.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:53:41.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35053/jwks.json with env {}
11:53:41.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:41.491106 3510 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.70.254" port: 44597 } has no permanent_uuid. Determining permanent_uuid...
11:53:41.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:41.516666 3510 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.70.253" port: 36585 } has no permanent_uuid. Determining permanent_uuid...
11:53:41.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:41.535665 3510 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.70.252" port: 34019 } has no permanent_uuid. Determining permanent_uuid...
11:53:41.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:41.538130 3462 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.70.252:34019: connect: Connection refused (error 111)
11:53:41.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:41.542594 3510 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.70.252:34019: Network error: Client connection negotiation failed: client connection to 127.3.70.252:34019: connect: Connection refused (error 111)
11:53:41.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:41.592134 3510 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.70.252" port: 34019 } attempt: 1
11:53:41.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:41.596939 3510 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.70.252:34019: Network error: Client connection negotiation failed: client connection to 127.3.70.252:34019: connect: Connection refused (error 111)
11:53:41.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:41.665431 3510 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.70.252" port: 34019 } attempt: 2
11:53:41.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:41.669723 3510 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.70.252:34019: Network error: Client connection negotiation failed: client connection to 127.3.70.252:34019: connect: Connection refused (error 111)
11:53:41.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:41.761149 3510 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.70.252" port: 34019 } attempt: 3
11:53:41.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:41.765574 3510 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.70.252:34019: Network error: Client connection negotiation failed: client connection to 127.3.70.252:34019: connect: Connection refused (error 111)
11:53:41.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:41.838549 3512 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:53:41.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:41.839107 3512 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:53:41.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:41.839560 3512 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:53:41.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:41.871021 3512 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
11:53:41.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:41.871332 3512 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:53:41.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:41.871518 3512 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
11:53:41.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:41.871768 3512 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
11:53:41.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:41.872063 3512 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
11:53:41.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:41.872238 3512 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35053/jwks.json
11:53:41.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:41.907513 3512 master_runner.cc:386] Master server non-default flags:
11:53:41.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:53:41.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.70.212:39605
11:53:41.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:53:41.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:53:41.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/master-2/data
11:53:41.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/master-2/wal
11:53:41.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:53:41.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.70.254:44597,127.3.70.253:36585,127.3.70.252:34019
11:53:41.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:53:41.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:53:41.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:53:41.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.70.252:34019
11:53:41.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:53:41.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:53:41.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:53:41.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35053/jwks.json
11:53:41.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:53:41.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:53:41.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/master-2/data/info.pb
11:53:41.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.70.252
11:53:41.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:53:41.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:53:41.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:53:41.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:53:41.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:53:41.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:53:41.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/master-2/logs
11:53:41.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:53:41.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:53:41.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:53:41.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
11:53:41.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:53:41.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:53:41.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:53:41.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 10:52:50 UTC on ccycloud.quasar-peqjre.root.comops.site
11:53:41.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679900
11:53:41.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:53:41.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:41.908926 3512 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:53:41.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:41.909384 3510 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.70.252" port: 34019 } attempt: 4
11:53:41.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:41.911187 3512 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:53:41.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:41.914642 3510 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.70.252:34019: Network error: Client connection negotiation failed: client connection to 127.3.70.252:34019: connect: Connection refused (error 111)
11:53:41.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:41.931787 3523 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:53:41.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:41.947150 3435 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.70.253" port: 36585 } attempt: 7
11:53:41.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:41.965163 3435 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.70.252" port: 34019 } has no permanent_uuid. Determining permanent_uuid...
11:53:41.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:41.974206 3435 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.70.252:34019: Network error: Client connection negotiation failed: client connection to 127.3.70.252:34019: connect: Connection refused (error 111)
11:53:42.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:42.032725 3435 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.70.252" port: 34019 } attempt: 1
11:53:42.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:42.036855 3435 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.70.252:34019: Network error: Client connection negotiation failed: client connection to 127.3.70.252:34019: connect: Connection refused (error 111)
11:53:42.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:42.118618 3435 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.70.252" port: 34019 } attempt: 2
11:53:42.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:42.123080 3435 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.70.252:34019: Network error: Client connection negotiation failed: client connection to 127.3.70.252:34019: connect: Connection refused (error 111)
11:53:42.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:42.208599 3435 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.70.252" port: 34019 } attempt: 3
11:53:42.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:42.214073 3435 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.70.252:34019: Network error: Client connection negotiation failed: client connection to 127.3.70.252:34019: connect: Connection refused (error 111)
11:53:42.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:42.214144 3510 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.70.252" port: 34019 } attempt: 5
11:53:42.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:42.219285 3510 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.70.252:34019: Network error: Client connection negotiation failed: client connection to 127.3.70.252:34019: connect: Connection refused (error 111)
11:53:42.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:42.354700 3435 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.70.252" port: 34019 } attempt: 4
11:53:42.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:42.359690 3435 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.70.252:34019: Network error: Client connection negotiation failed: client connection to 127.3.70.252:34019: connect: Connection refused (error 111)
11:53:42.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:42.643118 3435 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.70.252" port: 34019 } attempt: 5
11:53:42.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:42.647917 3435 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.70.252:34019: Network error: Client connection negotiation failed: client connection to 127.3.70.252:34019: connect: Connection refused (error 111)
11:53:42.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:42.766714 3510 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.70.252" port: 34019 } attempt: 6
11:53:42.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:42.776199 3510 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.70.252:34019: Network error: Client connection negotiation failed: client connection to 127.3.70.252:34019: connect: Connection refused (error 111)
11:53:43.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:41.932256 3522 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:53:43.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:41.932484 3525 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:53:43.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:43.194901 3524 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
11:53:43.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:43.195024 3512 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
11:53:43.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:43.199525 3512 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:53:43.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:43.200320 3435 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.70.252" port: 34019 } attempt: 6
11:53:43.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:43.204067 3512 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:53:43.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:43.205549 3512 hybrid_clock.cc:648] HybridClock initialized: now 1790682823205501 us; error 55 us; skew 500 ppm
11:53:43.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:43.206481 3512 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:53:43.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:43.208731 3435 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.70.252:34019: Network error: Client connection negotiation failed: client connection to 127.3.70.252:34019: connect: Connection refused (error 111)
11:53:43.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:43.218449 3512 webserver.cc:460] Webserver started at http://127.3.70.252:42081/ using document root <none> and password file <none>
11:53:43.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:43.219725 3512 fs_manager.cc:359] Metadata directory not provided
11:53:43.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:43.220019 3512 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:53:43.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:43.220564 3512 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
11:53:43.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:43.227737 3512 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/master-2/data/instance:
11:53:43.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4f9253da5bd24ea093bd45e0bbc731f5"
11:53:43.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:53:43 on dist-test-slave-cvvz"
11:53:43.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:43.229486 3512 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/master-2/wal/instance:
11:53:43.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4f9253da5bd24ea093bd45e0bbc731f5"
11:53:43.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:53:43 on dist-test-slave-cvvz"
11:53:43.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:43.239262 3512 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.007s sys 0.001s
11:53:43.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:43.246788 3538 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:53:43.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:43.248164 3512 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
11:53:43.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:43.248598 3512 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/master-2/data,/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/master-2/wal
11:53:43.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4f9253da5bd24ea093bd45e0bbc731f5"
11:53:43.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:53:43 on dist-test-slave-cvvz"
11:53:43.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:43.248942 3512 fs_report.cc:383] FS layout report
11:53:43.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:53:43.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/master-2/wal
11:53:43.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/master-2/wal
11:53:43.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/master-2/data/data
11:53:43.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:53:43.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:53:43.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:53:43.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:53:43.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:53:43.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:53:43.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:53:43.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:53:43.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:53:43.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:53:43.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:53:43.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:43.301617 3512 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:53:43.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:43.303241 3512 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:53:43.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:43.303694 3512 kserver.cc:163] Server-wide thread pool size limit: 3276
11:53:43.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:43.384941 3512 rpc_server.cc:307] RPC server started. Bound to: 127.3.70.252:34019
11:53:43.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:43.385018 3589 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.3.70.252:34019 every 8 connection(s)
11:53:43.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:43.387988 3512 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/master-2/data/info.pb
11:53:43.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:43.391624 3355 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskrtJWFm/build/tsan/bin/kudu as pid 3512
11:53:43.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:43.392135 3355 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/master-2/wal/instance
11:53:43.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:43.394898 3590 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
11:53:43.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:43.414106 3590 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.70.254" port: 44597 } has no permanent_uuid. Determining permanent_uuid...
11:53:43.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:43.442629 3590 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.70.253" port: 36585 } has no permanent_uuid. Determining permanent_uuid...
11:53:43.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:43.454885 3590 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.70.252" port: 34019 } has no permanent_uuid. Determining permanent_uuid...
11:53:43.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:43.475730 3590 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4f9253da5bd24ea093bd45e0bbc731f5: Bootstrap starting.
11:53:43.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:43.482101 3590 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 4f9253da5bd24ea093bd45e0bbc731f5: Neither blocks nor log segments found. Creating new log.
11:53:43.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:43.484490 3590 log.cc:824] T 00000000000000000000000000000000 P 4f9253da5bd24ea093bd45e0bbc731f5: Log is configured to *not* fsync() on all Append() calls
11:53:43.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:43.490330 3590 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4f9253da5bd24ea093bd45e0bbc731f5: No bootstrap required, opened a new log
11:53:43.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:43.511806 3590 raft_consensus.cc:348] T 00000000000000000000000000000000 P 4f9253da5bd24ea093bd45e0bbc731f5 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73746926f8964f6590c9b9277e05f3bc" member_type: VOTER last_known_addr { host: "127.3.70.254" port: 44597 } } peers { permanent_uuid: "5356a27f5bb44efaacb57e10ccc592ec" member_type: VOTER last_known_addr { host: "127.3.70.253" port: 36585 } } peers { permanent_uuid: "4f9253da5bd24ea093bd45e0bbc731f5" member_type: VOTER last_known_addr { host: "127.3.70.252" port: 34019 } }
11:53:43.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:43.512501 3590 raft_consensus.cc:374] T 00000000000000000000000000000000 P 4f9253da5bd24ea093bd45e0bbc731f5 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
11:53:43.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:43.512763 3590 raft_consensus.cc:729] T 00000000000000000000000000000000 P 4f9253da5bd24ea093bd45e0bbc731f5 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4f9253da5bd24ea093bd45e0bbc731f5, State: Initialized, Role: FOLLOWER
11:53:43.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:43.514055 3590 consensus_queue.cc:260] T 00000000000000000000000000000000 P 4f9253da5bd24ea093bd45e0bbc731f5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73746926f8964f6590c9b9277e05f3bc" member_type: VOTER last_known_addr { host: "127.3.70.254" port: 44597 } } peers { permanent_uuid: "5356a27f5bb44efaacb57e10ccc592ec" member_type: VOTER last_known_addr { host: "127.3.70.253" port: 36585 } } peers { permanent_uuid: "4f9253da5bd24ea093bd45e0bbc731f5" member_type: VOTER last_known_addr { host: "127.3.70.252" port: 34019 } }
11:53:43.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:43.516678 3599 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4f9253da5bd24ea093bd45e0bbc731f5 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73746926f8964f6590c9b9277e05f3bc" member_type: VOTER last_known_addr { host: "127.3.70.254" port: 44597 } } peers { permanent_uuid: "5356a27f5bb44efaacb57e10ccc592ec" member_type: VOTER last_known_addr { host: "127.3.70.253" port: 36585 } } peers { permanent_uuid: "4f9253da5bd24ea093bd45e0bbc731f5" member_type: VOTER last_known_addr { host: "127.3.70.252" port: 34019 } } }
11:53:43.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:43.517458 3599 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4f9253da5bd24ea093bd45e0bbc731f5 [sys.catalog]: This master's current role is: FOLLOWER
11:53:43.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:43.518123 3590 sys_catalog.cc:564] T 00000000000000000000000000000000 P 4f9253da5bd24ea093bd45e0bbc731f5 [sys.catalog]: configured and running, proceeding with master startup.
11:53:43.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:43.545353 3610 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4f9253da5bd24ea093bd45e0bbc731f5: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:53:43.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:43.545701 3610 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:53:43.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:43.559746 3544 tablet.cc:2369] T 00000000000000000000000000000000 P 4f9253da5bd24ea093bd45e0bbc731f5: Can't schedule compaction. Clean time has not been advanced past its initial value.
11:53:43.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:43.540283 3512 master_runner.cc:186] Error getting master registration for 127.3.70.254:44597: OK, instance_id { permanent_uuid: "73746926f8964f6590c9b9277e05f3bc" instance_seqno: 1790682819564674 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
11:53:43.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:43.565063 3512 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
11:53:43.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:43.836668 3510 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.70.252" port: 34019 } attempt: 7
11:53:43.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:43.857533 3510 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5356a27f5bb44efaacb57e10ccc592ec: Bootstrap starting.
11:53:43.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:43.863845 3510 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 5356a27f5bb44efaacb57e10ccc592ec: Neither blocks nor log segments found. Creating new log.
11:53:43.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:43.866350 3510 log.cc:824] T 00000000000000000000000000000000 P 5356a27f5bb44efaacb57e10ccc592ec: Log is configured to *not* fsync() on all Append() calls
11:53:43.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:43.873296 3510 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5356a27f5bb44efaacb57e10ccc592ec: No bootstrap required, opened a new log
11:53:43.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:43.894140 3510 raft_consensus.cc:348] T 00000000000000000000000000000000 P 5356a27f5bb44efaacb57e10ccc592ec [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73746926f8964f6590c9b9277e05f3bc" member_type: VOTER last_known_addr { host: "127.3.70.254" port: 44597 } } peers { permanent_uuid: "5356a27f5bb44efaacb57e10ccc592ec" member_type: VOTER last_known_addr { host: "127.3.70.253" port: 36585 } } peers { permanent_uuid: "4f9253da5bd24ea093bd45e0bbc731f5" member_type: VOTER last_known_addr { host: "127.3.70.252" port: 34019 } }
11:53:43.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:43.894843 3510 raft_consensus.cc:374] T 00000000000000000000000000000000 P 5356a27f5bb44efaacb57e10ccc592ec [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
11:53:43.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:43.895153 3510 raft_consensus.cc:729] T 00000000000000000000000000000000 P 5356a27f5bb44efaacb57e10ccc592ec [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5356a27f5bb44efaacb57e10ccc592ec, State: Initialized, Role: FOLLOWER
11:53:43.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:43.896310 3510 consensus_queue.cc:260] T 00000000000000000000000000000000 P 5356a27f5bb44efaacb57e10ccc592ec [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73746926f8964f6590c9b9277e05f3bc" member_type: VOTER last_known_addr { host: "127.3.70.254" port: 44597 } } peers { permanent_uuid: "5356a27f5bb44efaacb57e10ccc592ec" member_type: VOTER last_known_addr { host: "127.3.70.253" port: 36585 } } peers { permanent_uuid: "4f9253da5bd24ea093bd45e0bbc731f5" member_type: VOTER last_known_addr { host: "127.3.70.252" port: 34019 } }
11:53:43.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:43.898996 3614 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5356a27f5bb44efaacb57e10ccc592ec [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73746926f8964f6590c9b9277e05f3bc" member_type: VOTER last_known_addr { host: "127.3.70.254" port: 44597 } } peers { permanent_uuid: "5356a27f5bb44efaacb57e10ccc592ec" member_type: VOTER last_known_addr { host: "127.3.70.253" port: 36585 } } peers { permanent_uuid: "4f9253da5bd24ea093bd45e0bbc731f5" member_type: VOTER last_known_addr { host: "127.3.70.252" port: 34019 } } }
11:53:43.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:43.899864 3614 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5356a27f5bb44efaacb57e10ccc592ec [sys.catalog]: This master's current role is: FOLLOWER
11:53:43.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:43.900533 3510 sys_catalog.cc:564] T 00000000000000000000000000000000 P 5356a27f5bb44efaacb57e10ccc592ec [sys.catalog]: configured and running, proceeding with master startup.
11:53:43.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:43.933208 3625 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5356a27f5bb44efaacb57e10ccc592ec: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:53:43.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:43.933574 3625 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:53:43.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:43.927531 3437 master_runner.cc:186] Error getting master registration for 127.3.70.254:44597: OK, instance_id { permanent_uuid: "73746926f8964f6590c9b9277e05f3bc" instance_seqno: 1790682819564674 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
11:53:43.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:43.961984 3437 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
11:53:44.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.006098 3599 raft_consensus.cc:482] T 00000000000000000000000000000000 P 4f9253da5bd24ea093bd45e0bbc731f5 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
11:53:44.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.006687 3599 raft_consensus.cc:504] T 00000000000000000000000000000000 P 4f9253da5bd24ea093bd45e0bbc731f5 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73746926f8964f6590c9b9277e05f3bc" member_type: VOTER last_known_addr { host: "127.3.70.254" port: 44597 } } peers { permanent_uuid: "5356a27f5bb44efaacb57e10ccc592ec" member_type: VOTER last_known_addr { host: "127.3.70.253" port: 36585 } } peers { permanent_uuid: "4f9253da5bd24ea093bd45e0bbc731f5" member_type: VOTER last_known_addr { host: "127.3.70.252" port: 34019 } }
11:53:44.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.009512 3599 leader_election.cc:288] T 00000000000000000000000000000000 P 4f9253da5bd24ea093bd45e0bbc731f5 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 73746926f8964f6590c9b9277e05f3bc (127.3.70.254:44597), 5356a27f5bb44efaacb57e10ccc592ec (127.3.70.253:36585)
11:53:44.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.012305 3485 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4f9253da5bd24ea093bd45e0bbc731f5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5356a27f5bb44efaacb57e10ccc592ec" is_pre_election: true
11:53:44.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.013279 3485 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 5356a27f5bb44efaacb57e10ccc592ec [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4f9253da5bd24ea093bd45e0bbc731f5 in term 0.
11:53:44.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.014822 3540 leader_election.cc:302] T 00000000000000000000000000000000 P 4f9253da5bd24ea093bd45e0bbc731f5 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4f9253da5bd24ea093bd45e0bbc731f5, 5356a27f5bb44efaacb57e10ccc592ec; no voters:
11:53:44.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.015823 3599 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 4f9253da5bd24ea093bd45e0bbc731f5 [term 0 FOLLOWER]: Leader pre-election won for term 1
11:53:44.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.016157 3599 raft_consensus.cc:482] T 00000000000000000000000000000000 P 4f9253da5bd24ea093bd45e0bbc731f5 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
11:53:44.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.016492 3599 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 4f9253da5bd24ea093bd45e0bbc731f5 [term 0 FOLLOWER]: Advancing to term 1
11:53:44.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.024112 3599 raft_consensus.cc:504] T 00000000000000000000000000000000 P 4f9253da5bd24ea093bd45e0bbc731f5 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73746926f8964f6590c9b9277e05f3bc" member_type: VOTER last_known_addr { host: "127.3.70.254" port: 44597 } } peers { permanent_uuid: "5356a27f5bb44efaacb57e10ccc592ec" member_type: VOTER last_known_addr { host: "127.3.70.253" port: 36585 } } peers { permanent_uuid: "4f9253da5bd24ea093bd45e0bbc731f5" member_type: VOTER last_known_addr { host: "127.3.70.252" port: 34019 } }
11:53:44.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.026218 3599 leader_election.cc:288] T 00000000000000000000000000000000 P 4f9253da5bd24ea093bd45e0bbc731f5 [CANDIDATE]: Term 1 election: Requested vote from peers 73746926f8964f6590c9b9277e05f3bc (127.3.70.254:44597), 5356a27f5bb44efaacb57e10ccc592ec (127.3.70.253:36585)
11:53:44.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.027704 3485 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4f9253da5bd24ea093bd45e0bbc731f5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5356a27f5bb44efaacb57e10ccc592ec"
11:53:44.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.028259 3485 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 5356a27f5bb44efaacb57e10ccc592ec [term 0 FOLLOWER]: Advancing to term 1
11:53:44.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.039181 3485 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 5356a27f5bb44efaacb57e10ccc592ec [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4f9253da5bd24ea093bd45e0bbc731f5 in term 1.
11:53:44.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.040742 3540 leader_election.cc:302] T 00000000000000000000000000000000 P 4f9253da5bd24ea093bd45e0bbc731f5 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4f9253da5bd24ea093bd45e0bbc731f5, 5356a27f5bb44efaacb57e10ccc592ec; no voters:
11:53:44.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.041667 3599 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 4f9253da5bd24ea093bd45e0bbc731f5 [term 1 FOLLOWER]: Leader election won for term 1
11:53:44.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.052361 3599 raft_consensus.cc:686] T 00000000000000000000000000000000 P 4f9253da5bd24ea093bd45e0bbc731f5 [term 1 LEADER]: Becoming Leader. State: Replica: 4f9253da5bd24ea093bd45e0bbc731f5, State: Running, Role: LEADER
11:53:44.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.054003 3599 consensus_queue.cc:237] T 00000000000000000000000000000000 P 4f9253da5bd24ea093bd45e0bbc731f5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73746926f8964f6590c9b9277e05f3bc" member_type: VOTER last_known_addr { host: "127.3.70.254" port: 44597 } } peers { permanent_uuid: "5356a27f5bb44efaacb57e10ccc592ec" member_type: VOTER last_known_addr { host: "127.3.70.253" port: 36585 } } peers { permanent_uuid: "4f9253da5bd24ea093bd45e0bbc731f5" member_type: VOTER last_known_addr { host: "127.3.70.252" port: 34019 } }
11:53:44.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.070454 3627 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4f9253da5bd24ea093bd45e0bbc731f5 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4f9253da5bd24ea093bd45e0bbc731f5. Latest consensus state: current_term: 1 leader_uuid: "4f9253da5bd24ea093bd45e0bbc731f5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73746926f8964f6590c9b9277e05f3bc" member_type: VOTER last_known_addr { host: "127.3.70.254" port: 44597 } } peers { permanent_uuid: "5356a27f5bb44efaacb57e10ccc592ec" member_type: VOTER last_known_addr { host: "127.3.70.253" port: 36585 } } peers { permanent_uuid: "4f9253da5bd24ea093bd45e0bbc731f5" member_type: VOTER last_known_addr { host: "127.3.70.252" port: 34019 } } }
11:53:44.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.071273 3627 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4f9253da5bd24ea093bd45e0bbc731f5 [sys.catalog]: This master's current role is: LEADER
11:53:44.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.074761 3629 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
11:53:44.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.082100 3629 catalog_manager.cc:1484] Initializing Kudu cluster ID...
11:53:44.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.099965 3485 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 5356a27f5bb44efaacb57e10ccc592ec [term 1 FOLLOWER]: Refusing update from remote peer 4f9253da5bd24ea093bd45e0bbc731f5: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
11:53:44.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.101774 3627 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 4f9253da5bd24ea093bd45e0bbc731f5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5356a27f5bb44efaacb57e10ccc592ec" member_type: VOTER last_known_addr { host: "127.3.70.253" port: 36585 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
11:53:44.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:44.146027 3464 tablet.cc:2369] T 00000000000000000000000000000000 P 5356a27f5bb44efaacb57e10ccc592ec: Can't schedule compaction. Clean time has not been advanced past its initial value.
11:53:44.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.150195 3614 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5356a27f5bb44efaacb57e10ccc592ec [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4f9253da5bd24ea093bd45e0bbc731f5. Latest consensus state: current_term: 1 leader_uuid: "4f9253da5bd24ea093bd45e0bbc731f5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73746926f8964f6590c9b9277e05f3bc" member_type: VOTER last_known_addr { host: "127.3.70.254" port: 44597 } } peers { permanent_uuid: "5356a27f5bb44efaacb57e10ccc592ec" member_type: VOTER last_known_addr { host: "127.3.70.253" port: 36585 } } peers { permanent_uuid: "4f9253da5bd24ea093bd45e0bbc731f5" member_type: VOTER last_known_addr { host: "127.3.70.252" port: 34019 } } }
11:53:44.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.151037 3614 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5356a27f5bb44efaacb57e10ccc592ec [sys.catalog]: This master's current role is: FOLLOWER
11:53:44.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.155987 3630 mvcc.cc:205] Tried to move back new op lower bound from 7334636847497175040 to 7334636847338041344. Current Snapshot: MvccSnapshot[applied={T|T < 7334636847497175040}]
11:53:44.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.157151 3631 mvcc.cc:205] Tried to move back new op lower bound from 7334636847497175040 to 7334636847338041344. Current Snapshot: MvccSnapshot[applied={T|T < 7334636847497175040}]
11:53:44.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.159680 3627 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4f9253da5bd24ea093bd45e0bbc731f5 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "4f9253da5bd24ea093bd45e0bbc731f5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73746926f8964f6590c9b9277e05f3bc" member_type: VOTER last_known_addr { host: "127.3.70.254" port: 44597 } } peers { permanent_uuid: "5356a27f5bb44efaacb57e10ccc592ec" member_type: VOTER last_known_addr { host: "127.3.70.253" port: 36585 } } peers { permanent_uuid: "4f9253da5bd24ea093bd45e0bbc731f5" member_type: VOTER last_known_addr { host: "127.3.70.252" port: 34019 } } }
11:53:44.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.160434 3627 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4f9253da5bd24ea093bd45e0bbc731f5 [sys.catalog]: This master's current role is: LEADER
11:53:44.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.162474 3629 catalog_manager.cc:1347] Generated new cluster ID: 9c29613cf3944d04bb8588c2502847d0
11:53:44.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.162757 3629 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
11:53:44.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.174855 3614 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5356a27f5bb44efaacb57e10ccc592ec [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "4f9253da5bd24ea093bd45e0bbc731f5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73746926f8964f6590c9b9277e05f3bc" member_type: VOTER last_known_addr { host: "127.3.70.254" port: 44597 } } peers { permanent_uuid: "5356a27f5bb44efaacb57e10ccc592ec" member_type: VOTER last_known_addr { host: "127.3.70.253" port: 36585 } } peers { permanent_uuid: "4f9253da5bd24ea093bd45e0bbc731f5" member_type: VOTER last_known_addr { host: "127.3.70.252" port: 34019 } } }
11:53:44.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.175721 3614 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5356a27f5bb44efaacb57e10ccc592ec [sys.catalog]: This master's current role is: FOLLOWER
11:53:44.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.233428 3629 catalog_manager.cc:1370] Generated new certificate authority record
11:53:44.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.240454 3629 catalog_manager.cc:1504] Loading token signing keys...
11:53:44.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.242147 3435 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.70.252" port: 34019 } attempt: 7
11:53:44.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.278987 3629 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 4f9253da5bd24ea093bd45e0bbc731f5: Generated new TSK 0
11:53:44.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.280268 3629 catalog_manager.cc:1514] Initializing in-progress tserver states...
11:53:44.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.293926 3435 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 73746926f8964f6590c9b9277e05f3bc: Bootstrap starting.
11:53:44.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.302130 3435 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 73746926f8964f6590c9b9277e05f3bc: Neither blocks nor log segments found. Creating new log.
11:53:44.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.305119 3435 log.cc:824] T 00000000000000000000000000000000 P 73746926f8964f6590c9b9277e05f3bc: Log is configured to *not* fsync() on all Append() calls
11:53:44.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.319088 3435 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 73746926f8964f6590c9b9277e05f3bc: No bootstrap required, opened a new log
11:53:44.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.321323 3355 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskrtJWFm/build/tsan/bin/kudu
11:53:44.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrtJWFm/build/tsan/bin/kudu
11:53:44.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/ts-0/wal
11:53:44.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/ts-0/data
11:53:44.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:53:44.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:53:44.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:53:44.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:53:44.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:53:44.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:53:44.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/ts-0/logs
11:53:44.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/ts-0/data/info.pb
11:53:44.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:53:44.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:53:44.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:53:44.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:53:44.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:53:44.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:53:44.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:53:44.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:53:44.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
11:53:44.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:53:44.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.70.193:0
11:53:44.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.70.193
11:53:44.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.70.193
11:53:44.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:53:44.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.70.254:44597,127.3.70.253:36585,127.3.70.252:34019
11:53:44.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.70.212:39605
11:53:44.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:53:44.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:53:44.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
11:53:44.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.414340 3435 raft_consensus.cc:348] T 00000000000000000000000000000000 P 73746926f8964f6590c9b9277e05f3bc [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73746926f8964f6590c9b9277e05f3bc" member_type: VOTER last_known_addr { host: "127.3.70.254" port: 44597 } } peers { permanent_uuid: "5356a27f5bb44efaacb57e10ccc592ec" member_type: VOTER last_known_addr { host: "127.3.70.253" port: 36585 } } peers { permanent_uuid: "4f9253da5bd24ea093bd45e0bbc731f5" member_type: VOTER last_known_addr { host: "127.3.70.252" port: 34019 } }
11:53:44.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.416884 3435 raft_consensus.cc:374] T 00000000000000000000000000000000 P 73746926f8964f6590c9b9277e05f3bc [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
11:53:44.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.418004 3435 raft_consensus.cc:729] T 00000000000000000000000000000000 P 73746926f8964f6590c9b9277e05f3bc [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 73746926f8964f6590c9b9277e05f3bc, State: Initialized, Role: FOLLOWER
11:53:44.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.422785 3435 consensus_queue.cc:260] T 00000000000000000000000000000000 P 73746926f8964f6590c9b9277e05f3bc [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73746926f8964f6590c9b9277e05f3bc" member_type: VOTER last_known_addr { host: "127.3.70.254" port: 44597 } } peers { permanent_uuid: "5356a27f5bb44efaacb57e10ccc592ec" member_type: VOTER last_known_addr { host: "127.3.70.253" port: 36585 } } peers { permanent_uuid: "4f9253da5bd24ea093bd45e0bbc731f5" member_type: VOTER last_known_addr { host: "127.3.70.252" port: 34019 } }
11:53:44.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.449388 3409 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4f9253da5bd24ea093bd45e0bbc731f5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "73746926f8964f6590c9b9277e05f3bc"
11:53:44.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.450265 3409 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 73746926f8964f6590c9b9277e05f3bc [term 0 FOLLOWER]: Advancing to term 1
11:53:44.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.453510 3410 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4f9253da5bd24ea093bd45e0bbc731f5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "73746926f8964f6590c9b9277e05f3bc" is_pre_election: true
11:53:44.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.461628 3409 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 73746926f8964f6590c9b9277e05f3bc [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4f9253da5bd24ea093bd45e0bbc731f5 in term 1.
11:53:44.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.464761 3408 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 73746926f8964f6590c9b9277e05f3bc [term 1 FOLLOWER]: Refusing update from remote peer 4f9253da5bd24ea093bd45e0bbc731f5: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
11:53:44.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.471597 3599 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 4f9253da5bd24ea093bd45e0bbc731f5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "73746926f8964f6590c9b9277e05f3bc" member_type: VOTER last_known_addr { host: "127.3.70.254" port: 44597 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
11:53:44.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.462715 3435 sys_catalog.cc:564] T 00000000000000000000000000000000 P 73746926f8964f6590c9b9277e05f3bc [sys.catalog]: configured and running, proceeding with master startup.
11:53:44.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.447445 3638 sys_catalog.cc:455] T 00000000000000000000000000000000 P 73746926f8964f6590c9b9277e05f3bc [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73746926f8964f6590c9b9277e05f3bc" member_type: VOTER last_known_addr { host: "127.3.70.254" port: 44597 } } peers { permanent_uuid: "5356a27f5bb44efaacb57e10ccc592ec" member_type: VOTER last_known_addr { host: "127.3.70.253" port: 36585 } } peers { permanent_uuid: "4f9253da5bd24ea093bd45e0bbc731f5" member_type: VOTER last_known_addr { host: "127.3.70.252" port: 34019 } } }
11:53:44.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.483685 3638 sys_catalog.cc:458] T 00000000000000000000000000000000 P 73746926f8964f6590c9b9277e05f3bc [sys.catalog]: This master's current role is: FOLLOWER
11:53:44.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:44.520010 3389 tablet.cc:2369] T 00000000000000000000000000000000 P 73746926f8964f6590c9b9277e05f3bc: Can't schedule compaction. Clean time has not been advanced past its initial value.
11:53:44.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.558689 3638 sys_catalog.cc:455] T 00000000000000000000000000000000 P 73746926f8964f6590c9b9277e05f3bc [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4f9253da5bd24ea093bd45e0bbc731f5. Latest consensus state: current_term: 1 leader_uuid: "4f9253da5bd24ea093bd45e0bbc731f5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73746926f8964f6590c9b9277e05f3bc" member_type: VOTER last_known_addr { host: "127.3.70.254" port: 44597 } } peers { permanent_uuid: "5356a27f5bb44efaacb57e10ccc592ec" member_type: VOTER last_known_addr { host: "127.3.70.253" port: 36585 } } peers { permanent_uuid: "4f9253da5bd24ea093bd45e0bbc731f5" member_type: VOTER last_known_addr { host: "127.3.70.252" port: 34019 } } }
11:53:44.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.559512 3638 sys_catalog.cc:458] T 00000000000000000000000000000000 P 73746926f8964f6590c9b9277e05f3bc [sys.catalog]: This master's current role is: FOLLOWER
11:53:44.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.560573 3639 sys_catalog.cc:455] T 00000000000000000000000000000000 P 73746926f8964f6590c9b9277e05f3bc [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "4f9253da5bd24ea093bd45e0bbc731f5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73746926f8964f6590c9b9277e05f3bc" member_type: VOTER last_known_addr { host: "127.3.70.254" port: 44597 } } peers { permanent_uuid: "5356a27f5bb44efaacb57e10ccc592ec" member_type: VOTER last_known_addr { host: "127.3.70.253" port: 36585 } } peers { permanent_uuid: "4f9253da5bd24ea093bd45e0bbc731f5" member_type: VOTER last_known_addr { host: "127.3.70.252" port: 34019 } } }
11:53:44.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.561503 3639 sys_catalog.cc:458] T 00000000000000000000000000000000 P 73746926f8964f6590c9b9277e05f3bc [sys.catalog]: This master's current role is: FOLLOWER
11:53:44.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.640053 3655 catalog_manager.cc:1259] Loaded cluster ID: 9c29613cf3944d04bb8588c2502847d0
11:53:44.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.640391 3655 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 73746926f8964f6590c9b9277e05f3bc: loading cluster ID for follower catalog manager: success
11:53:44.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.672909 3655 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 73746926f8964f6590c9b9277e05f3bc: acquiring CA information for follower catalog manager: success
11:53:44.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.684881 3655 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 73746926f8964f6590c9b9277e05f3bc: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
11:53:44.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.900120 3599 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4f9253da5bd24ea093bd45e0bbc731f5 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "4f9253da5bd24ea093bd45e0bbc731f5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "73746926f8964f6590c9b9277e05f3bc" member_type: VOTER last_known_addr { host: "127.3.70.254" port: 44597 } } peers { permanent_uuid: "5356a27f5bb44efaacb57e10ccc592ec" member_type: VOTER last_known_addr { host: "127.3.70.253" port: 36585 } } peers { permanent_uuid: "4f9253da5bd24ea093bd45e0bbc731f5" member_type: VOTER last_known_addr { host: "127.3.70.252" port: 34019 } } }
11:53:44.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.901312 3599 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4f9253da5bd24ea093bd45e0bbc731f5 [sys.catalog]: This master's current role is: LEADER
11:53:44.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:44.902904 3637 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:53:44.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:44.903776 3637 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:53:44.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:44.904707 3637 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:53:44.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.939612 3625 catalog_manager.cc:1259] Loaded cluster ID: 9c29613cf3944d04bb8588c2502847d0
11:53:44.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.940287 3625 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 5356a27f5bb44efaacb57e10ccc592ec: loading cluster ID for follower catalog manager: success
11:53:44.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.955715 3625 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 5356a27f5bb44efaacb57e10ccc592ec: acquiring CA information for follower catalog manager: success
11:53:44.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:44.960536 3637 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:53:44.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:44.962314 3637 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.3.70.193
11:53:44.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:44.969722 3625 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 5356a27f5bb44efaacb57e10ccc592ec: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
11:53:45.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:45.024828 3637 tablet_server_runner.cc:78] Tablet server non-default flags:
11:53:45.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:53:45.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.70.212:39605
11:53:45.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:53:45.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:53:45.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/ts-0/data
11:53:45.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/ts-0/wal
11:53:45.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:53:45.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:53:45.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.70.193:0
11:53:45.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:53:45.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:53:45.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:53:45.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/ts-0/data/info.pb
11:53:45.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.70.193
11:53:45.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:53:45.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.70.254:44597,127.3.70.253:36585,127.3.70.252:34019
11:53:45.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:53:45.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:53:45.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:53:45.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:53:45.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:53:45.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.70.193
11:53:45.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/ts-0/logs
11:53:45.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:53:45.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:53:45.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:53:45.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
11:53:45.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:53:45.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:53:45.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:53:45.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 10:52:50 UTC on ccycloud.quasar-peqjre.root.comops.site
11:53:45.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679900
11:53:45.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:53:45.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:45.032764 3637 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:53:45.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:45.035579 3637 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:53:45.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:45.080415 3665 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:53:47.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:45.084579 3666 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:53:47.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:47.286700 3637 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 2.199s user 0.248s sys 0.595s
11:53:47.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:47.287055 3637 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 2.199s user 0.248s sys 0.595s
11:53:47.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:46.459082 3664 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 3637
11:53:47.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:47.290266 3669 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:53:47.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:47.293682 3637 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
11:53:47.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:47.294162 3667 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2205 milliseconds
11:53:47.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:47.295581 3637 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:53:47.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:47.304369 3637 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:53:47.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:47.305853 3637 hybrid_clock.cc:648] HybridClock initialized: now 1790682827305805 us; error 60 us; skew 500 ppm
11:53:47.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:47.306938 3637 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:53:47.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:47.342075 3637 webserver.cc:460] Webserver started at http://127.3.70.193:36223/ using document root <none> and password file <none>
11:53:47.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:47.343571 3637 fs_manager.cc:359] Metadata directory not provided
11:53:47.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:47.343879 3637 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:53:47.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:47.344388 3637 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
11:53:47.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:47.351653 3637 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/ts-0/data/instance:
11:53:47.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7c59dc9dcb584104bfd80f414bdd2c88"
11:53:47.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:53:47 on dist-test-slave-cvvz"
11:53:47.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:47.353405 3637 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/ts-0/wal/instance:
11:53:47.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7c59dc9dcb584104bfd80f414bdd2c88"
11:53:47.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:53:47 on dist-test-slave-cvvz"
11:53:47.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:47.368763 3637 fs_manager.cc:691] Time spent creating directory manager: real 0.015s user 0.009s sys 0.005s
11:53:47.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:47.376982 3677 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:53:47.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:47.378588 3637 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.002s sys 0.002s
11:53:47.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:47.379079 3637 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/ts-0/data,/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/ts-0/wal
11:53:47.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7c59dc9dcb584104bfd80f414bdd2c88"
11:53:47.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:53:47 on dist-test-slave-cvvz"
11:53:47.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:47.379503 3637 fs_report.cc:383] FS layout report
11:53:47.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:53:47.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/ts-0/wal
11:53:47.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/ts-0/wal
11:53:47.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/ts-0/data/data
11:53:47.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:53:47.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:53:47.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:53:47.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:53:47.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:53:47.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:53:47.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:53:47.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:53:47.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:53:47.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:53:47.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:53:47.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:47.437304 3637 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:53:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:47.438930 3637 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:53:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:47.439499 3637 kserver.cc:163] Server-wide thread pool size limit: 3276
11:53:47.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:47.444934 3637 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
11:53:47.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:47.450188 3637 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
11:53:47.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:47.450438 3637 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
11:53:47.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:47.450733 3637 ts_tablet_manager.cc:610] Registered 0 tablets
11:53:47.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:47.450945 3637 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
11:53:47.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:47.782912 3637 rpc_server.cc:307] RPC server started. Bound to: 127.3.70.193:41409
11:53:47.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:47.783063 3791 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.3.70.193:41409 every 8 connection(s)
11:53:47.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:47.785964 3637 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/ts-0/data/info.pb
11:53:47.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:47.795816 3355 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskrtJWFm/build/tsan/bin/kudu as pid 3637
11:53:47.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:47.796770 3355 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/ts-0/wal/instance
11:53:47.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:47.839811 3355 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskrtJWFm/build/tsan/bin/kudu
11:53:47.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrtJWFm/build/tsan/bin/kudu
11:53:47.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/ts-1/wal
11:53:47.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/ts-1/data
11:53:47.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:53:47.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:53:47.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:53:47.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:53:47.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:53:47.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:53:47.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/ts-1/logs
11:53:47.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/ts-1/data/info.pb
11:53:47.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:53:47.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:53:47.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:53:47.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:53:47.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:53:47.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:53:47.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:53:47.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:53:47.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
11:53:47.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:53:47.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.70.194:0
11:53:47.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.70.194
11:53:47.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.70.194
11:53:47.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:53:47.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.70.254:44597,127.3.70.253:36585,127.3.70.252:34019
11:53:47.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.70.212:39605
11:53:47.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:53:47.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:53:47.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
11:53:47.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:47.842013 3794 heartbeater.cc:347] Connected to a master server at 127.3.70.254:44597
11:53:47.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:47.842514 3794 heartbeater.cc:464] Registering TS with master...
11:53:47.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:47.843967 3794 heartbeater.cc:511] Master 127.3.70.254:44597 requested a full tablet report, sending...
11:53:47.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:47.849392 3400 ts_manager.cc:194] Registered new tserver with Master: 7c59dc9dcb584104bfd80f414bdd2c88 (127.3.70.193:41409)
11:53:47.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:47.852355 3792 heartbeater.cc:347] Connected to a master server at 127.3.70.252:34019
11:53:47.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:47.852742 3792 heartbeater.cc:464] Registering TS with master...
11:53:47.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:47.853605 3792 heartbeater.cc:511] Master 127.3.70.252:34019 requested a full tablet report, sending...
11:53:47.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:47.856534 3555 ts_manager.cc:194] Registered new tserver with Master: 7c59dc9dcb584104bfd80f414bdd2c88 (127.3.70.193:41409)
11:53:47.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:47.859843 3555 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.3.70.193:51313
11:53:47.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:47.863986 3793 heartbeater.cc:347] Connected to a master server at 127.3.70.253:36585
11:53:47.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:47.864405 3793 heartbeater.cc:464] Registering TS with master...
11:53:47.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:47.865228 3793 heartbeater.cc:511] Master 127.3.70.253:36585 requested a full tablet report, sending...
11:53:47.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:47.868685 3475 ts_manager.cc:194] Registered new tserver with Master: 7c59dc9dcb584104bfd80f414bdd2c88 (127.3.70.193:41409)
11:53:48.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:48.182372 3802 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:53:48.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:48.182852 3802 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:53:48.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:48.183272 3802 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:53:48.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:48.215764 3802 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:53:48.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:48.216563 3802 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.3.70.194
11:53:48.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:48.253234 3802 tablet_server_runner.cc:78] Tablet server non-default flags:
11:53:48.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:53:48.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.70.212:39605
11:53:48.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:53:48.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:53:48.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/ts-1/data
11:53:48.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/ts-1/wal
11:53:48.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:53:48.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:53:48.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.70.194:0
11:53:48.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:53:48.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:53:48.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:53:48.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/ts-1/data/info.pb
11:53:48.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.70.194
11:53:48.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:53:48.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.70.254:44597,127.3.70.253:36585,127.3.70.252:34019
11:53:48.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:53:48.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:53:48.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:53:48.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:53:48.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:53:48.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.70.194
11:53:48.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/ts-1/logs
11:53:48.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:53:48.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:53:48.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:53:48.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
11:53:48.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:53:48.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:53:48.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:53:48.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 10:52:50 UTC on ccycloud.quasar-peqjre.root.comops.site
11:53:48.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679900
11:53:48.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:53:48.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:48.254400 3802 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:53:48.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:48.255901 3802 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:53:48.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:48.280588 3810 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:53:48.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:48.865623 3792 heartbeater.cc:503] Master 127.3.70.252:34019 was elected leader, sending a full tablet report...
11:53:49.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:48.281823 3809 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:53:49.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:48.281201 3812 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:53:49.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:48.282847 3802 server_base.cc:1034] running on GCE node
11:53:49.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:49.544601 3802 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:53:49.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:49.547497 3802 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:53:49.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:49.548954 3802 hybrid_clock.cc:648] HybridClock initialized: now 1790682829548896 us; error 74 us; skew 500 ppm
11:53:49.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:49.549855 3802 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:53:49.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:49.556824 3802 webserver.cc:460] Webserver started at http://127.3.70.194:44107/ using document root <none> and password file <none>
11:53:49.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:49.557750 3802 fs_manager.cc:359] Metadata directory not provided
11:53:49.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:49.557931 3802 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:53:49.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:49.558301 3802 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
11:53:49.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:49.563412 3802 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/ts-1/data/instance:
11:53:49.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "67ea8ffe14334b6c874c68196d63d1b2"
11:53:49.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:53:49 on dist-test-slave-cvvz"
11:53:49.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:49.564561 3802 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/ts-1/wal/instance:
11:53:49.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "67ea8ffe14334b6c874c68196d63d1b2"
11:53:49.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:53:49 on dist-test-slave-cvvz"
11:53:49.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:49.571858 3802 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.000s
11:53:49.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:49.577844 3819 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:53:49.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:49.578996 3802 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
11:53:49.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:49.579280 3802 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/ts-1/data,/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/ts-1/wal
11:53:49.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "67ea8ffe14334b6c874c68196d63d1b2"
11:53:49.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:53:49 on dist-test-slave-cvvz"
11:53:49.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:49.579648 3802 fs_report.cc:383] FS layout report
11:53:49.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:53:49.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/ts-1/wal
11:53:49.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/ts-1/wal
11:53:49.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/ts-1/data/data
11:53:49.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:53:49.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:53:49.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:53:49.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:53:49.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:53:49.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:53:49.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:53:49.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:53:49.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:53:49.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:53:49.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:53:49.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:49.629535 3802 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:53:49.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:49.631247 3802 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:53:49.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:49.631722 3802 kserver.cc:163] Server-wide thread pool size limit: 3276
11:53:49.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:49.634516 3802 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
11:53:49.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:49.639076 3802 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
11:53:49.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:49.639286 3802 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
11:53:49.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:49.639506 3802 ts_tablet_manager.cc:610] Registered 0 tablets
11:53:49.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:49.639647 3802 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
11:53:49.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:49.793020 3802 rpc_server.cc:307] RPC server started. Bound to: 127.3.70.194:44057
11:53:49.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:49.793191 3931 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.3.70.194:44057 every 8 connection(s)
11:53:49.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:49.795806 3802 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/ts-1/data/info.pb
11:53:49.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:49.801793 3355 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskrtJWFm/build/tsan/bin/kudu as pid 3802
11:53:49.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:49.802268 3355 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/ts-1/wal/instance
11:53:49.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:49.830837 3355 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskrtJWFm/build/tsan/bin/kudu
11:53:49.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrtJWFm/build/tsan/bin/kudu
11:53:49.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/ts-2/wal
11:53:49.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/ts-2/data
11:53:49.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:53:49.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:53:49.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:53:49.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:53:49.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:53:49.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:53:49.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/ts-2/logs
11:53:49.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/ts-2/data/info.pb
11:53:49.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:53:49.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:53:49.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:53:49.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:53:49.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:53:49.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:53:49.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:53:49.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:53:49.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
11:53:49.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:53:49.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.70.195:0
11:53:49.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.70.195
11:53:49.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.70.195
11:53:49.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:53:49.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.70.254:44597,127.3.70.253:36585,127.3.70.252:34019
11:53:49.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.70.212:39605
11:53:49.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:53:49.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:53:49.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
11:53:49.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:49.851495 3934 heartbeater.cc:347] Connected to a master server at 127.3.70.254:44597
11:53:49.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:49.851951 3934 heartbeater.cc:464] Registering TS with master...
11:53:49.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:49.853219 3934 heartbeater.cc:511] Master 127.3.70.254:44597 requested a full tablet report, sending...
11:53:49.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:49.857671 3400 ts_manager.cc:194] Registered new tserver with Master: 67ea8ffe14334b6c874c68196d63d1b2 (127.3.70.194:44057)
11:53:49.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:49.862107 3932 heartbeater.cc:347] Connected to a master server at 127.3.70.252:34019
11:53:49.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:49.862478 3932 heartbeater.cc:464] Registering TS with master...
11:53:49.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:49.863032 3933 heartbeater.cc:347] Connected to a master server at 127.3.70.253:36585
11:53:49.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:49.863396 3933 heartbeater.cc:464] Registering TS with master...
11:53:49.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:49.863436 3932 heartbeater.cc:511] Master 127.3.70.252:34019 requested a full tablet report, sending...
11:53:49.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:49.864210 3933 heartbeater.cc:511] Master 127.3.70.253:36585 requested a full tablet report, sending...
11:53:49.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:49.865809 3555 ts_manager.cc:194] Registered new tserver with Master: 67ea8ffe14334b6c874c68196d63d1b2 (127.3.70.194:44057)
11:53:49.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:49.866827 3475 ts_manager.cc:194] Registered new tserver with Master: 67ea8ffe14334b6c874c68196d63d1b2 (127.3.70.194:44057)
11:53:49.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:49.867622 3555 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.3.70.194:57095
11:53:50.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:50.158957 3942 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:53:50.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:50.159526 3942 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:53:50.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:50.160126 3942 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:53:50.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:50.191354 3942 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:53:50.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:50.192308 3942 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.3.70.195
11:53:50.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:50.228344 3942 tablet_server_runner.cc:78] Tablet server non-default flags:
11:53:50.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:53:50.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.70.212:39605
11:53:50.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:53:50.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:53:50.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/ts-2/data
11:53:50.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/ts-2/wal
11:53:50.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:53:50.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:53:50.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.70.195:0
11:53:50.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:53:50.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:53:50.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:53:50.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/ts-2/data/info.pb
11:53:50.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.70.195
11:53:50.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:53:50.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.70.254:44597,127.3.70.253:36585,127.3.70.252:34019
11:53:50.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:53:50.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:53:50.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:53:50.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:53:50.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:53:50.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.70.195
11:53:50.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/ts-2/logs
11:53:50.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:53:50.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:53:50.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:53:50.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
11:53:50.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:53:50.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:53:50.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:53:50.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 10:52:50 UTC on ccycloud.quasar-peqjre.root.comops.site
11:53:50.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679900
11:53:50.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:53:50.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:50.229974 3942 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:53:50.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:50.231817 3942 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:53:50.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:50.252174 3952 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:53:50.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:50.872114 3932 heartbeater.cc:503] Master 127.3.70.252:34019 was elected leader, sending a full tablet report...
11:53:51.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:50.252393 3950 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:53:51.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:50.253134 3949 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:53:51.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:51.503720 3951 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
11:53:51.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:51.503935 3942 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
11:53:51.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:51.508639 3942 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:53:51.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:51.511152 3942 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:53:51.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:51.512658 3942 hybrid_clock.cc:648] HybridClock initialized: now 1790682831512609 us; error 65 us; skew 500 ppm
11:53:51.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:51.514194 3942 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:53:51.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:51.521111 3942 webserver.cc:460] Webserver started at http://127.3.70.195:34295/ using document root <none> and password file <none>
11:53:51.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:51.522118 3942 fs_manager.cc:359] Metadata directory not provided
11:53:51.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:51.522373 3942 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:53:51.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:51.522907 3942 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
11:53:51.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:51.528154 3942 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/ts-2/data/instance:
11:53:51.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6be131ca93504b44b2d507e8cb1d3f88"
11:53:51.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:53:51 on dist-test-slave-cvvz"
11:53:51.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:51.529560 3942 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/ts-2/wal/instance:
11:53:51.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6be131ca93504b44b2d507e8cb1d3f88"
11:53:51.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:53:51 on dist-test-slave-cvvz"
11:53:51.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:51.537458 3942 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.007s sys 0.000s
11:53:51.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:51.543365 3959 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:53:51.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:51.544463 3942 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
11:53:51.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:51.544862 3942 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/ts-2/data,/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/ts-2/wal
11:53:51.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6be131ca93504b44b2d507e8cb1d3f88"
11:53:51.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:53:51 on dist-test-slave-cvvz"
11:53:51.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:51.545285 3942 fs_report.cc:383] FS layout report
11:53:51.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:53:51.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/ts-2/wal
11:53:51.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/ts-2/wal
11:53:51.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/ts-2/data/data
11:53:51.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:53:51.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:53:51.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:53:51.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:53:51.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:53:51.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:53:51.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:53:51.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:53:51.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:53:51.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:53:51.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:53:51.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:51.587090 3942 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:53:51.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:51.588645 3942 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:53:51.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:51.589195 3942 kserver.cc:163] Server-wide thread pool size limit: 3276
11:53:51.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:51.592262 3942 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
11:53:51.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:51.597342 3942 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
11:53:51.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:51.597601 3942 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
11:53:51.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:51.597903 3942 ts_tablet_manager.cc:610] Registered 0 tablets
11:53:51.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:51.598124 3942 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
11:53:51.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:51.760571 3942 rpc_server.cc:307] RPC server started. Bound to: 127.3.70.195:42721
11:53:51.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:51.760648 4071 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.3.70.195:42721 every 8 connection(s)
11:53:51.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:51.763623 3942 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/ts-2/data/info.pb
11:53:51.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:51.765882 3355 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskrtJWFm/build/tsan/bin/kudu as pid 3942
11:53:51.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:51.766527 3355 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster2521774542600998931/ts-2/wal/instance
11:53:51.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:51.823174 4073 heartbeater.cc:347] Connected to a master server at 127.3.70.253:36585
11:53:51.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:51.823220 4074 heartbeater.cc:347] Connected to a master server at 127.3.70.254:44597
11:53:51.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:51.823800 4073 heartbeater.cc:464] Registering TS with master...
11:53:51.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:51.823928 4074 heartbeater.cc:464] Registering TS with master...
11:53:51.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:51.825464 4073 heartbeater.cc:511] Master 127.3.70.253:36585 requested a full tablet report, sending...
11:53:51.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:51.825382 4074 heartbeater.cc:511] Master 127.3.70.254:44597 requested a full tablet report, sending...
11:53:51.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:51.831674 3399 ts_manager.cc:194] Registered new tserver with Master: 6be131ca93504b44b2d507e8cb1d3f88 (127.3.70.195:42721)
11:53:51.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:51.834409 3475 ts_manager.cc:194] Registered new tserver with Master: 6be131ca93504b44b2d507e8cb1d3f88 (127.3.70.195:42721)
11:53:51.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:51.848830 4072 heartbeater.cc:347] Connected to a master server at 127.3.70.252:34019
11:53:51.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:51.849227 4072 heartbeater.cc:464] Registering TS with master...
11:53:51.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:51.850103 4072 heartbeater.cc:511] Master 127.3.70.252:34019 requested a full tablet report, sending...
11:53:51.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:51.852612 3555 ts_manager.cc:194] Registered new tserver with Master: 6be131ca93504b44b2d507e8cb1d3f88 (127.3.70.195:42721)
11:53:51.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:51.855573 3555 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.3.70.195:52745
11:53:51.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:51.871017 3355 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
11:53:51.888 [DEBUG - main] (MiniKuduCluster.java:195) Response:
11:53:51.890 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
11:53:51.906 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.3.70.254"
port: 44597
}
bound_http_address {
host: "127.3.70.254"
port: 34711
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.3.70.253"
port: 36585
}
bound_http_address {
host: "127.3.70.253"
port: 33061
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.3.70.252"
port: 34019
}
bound_http_address {
host: "127.3.70.252"
port: 42081
}
}
}
11:53:51.913 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
11:53:51.916 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.3.70.193"
port: 41409
}
bound_http_address {
host: "127.3.70.193"
port: 36223
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.3.70.194"
port: 44057
}
bound_http_address {
host: "127.3.70.194"
port: 44107
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.3.70.195"
port: 42721
}
bound_http_address {
host: "127.3.70.195"
port: 34295
}
}
}
11:53:51.916 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
11:53:52.111 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
11:53:52.238 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskrtJWFm/test-tmp
11:53:52.240 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
11:53:52.240 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskrtJWFm/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
11:53:52.243 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
11:53:52.635 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:53:52.636 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
11:53:52.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:52.639364 4084 mini_oidc.cc:152] Starting JWKS server
11:53:52.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:52.643350 4084 webserver.cc:460] Webserver started at http://127.0.0.1:36819/ using document root <none> and password file <none>
11:53:52.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:52.643771 4084 mini_oidc.cc:189] Starting OIDC Discovery server
11:53:52.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:52.645565 4084 webserver.cc:460] Webserver started at http://127.0.0.1:42523/ using document root <none> and password file <none>
11:53:52.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:52.860937 4072 heartbeater.cc:503] Master 127.3.70.252:34019 was elected leader, sending a full tablet report...
11:53:54.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
11:53:54.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/krb5kdc/principal' for realm 'KRBTEST.COM',
11:53:54.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
11:53:54.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:53:54 dist-test-slave-cvvz krb5kdc[4097](info): setting up network...
11:53:54.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
11:53:54.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:53:54 dist-test-slave-cvvz krb5kdc[4097](info): set up 2 sockets
11:53:54.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:53:54 dist-test-slave-cvvz krb5kdc[4097](info): commencing operation
11:53:54.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
11:53:56.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:56.043385 4084 mini_kdc.cc:121] Time spent starting KDC: real 3.372s user 0.002s sys 0.024s
11:53:56.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
11:53:56.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
11:53:56.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
11:53:56.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:56.208489 4084 mini_kdc.cc:253] Time spent creating user principal test-admin: real 0.165s user 0.001s sys 0.004s
11:53:56.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
11:53:56.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:56.305347 3928 debug-util.cc:398] Leaking SignalData structure 0x7b08000b9520 after lost signal to thread 3804
11:53:56.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:56.306492 3928 debug-util.cc:398] Leaking SignalData structure 0x7b08000b9d40 after lost signal to thread 3931
11:53:56.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
11:53:56.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
11:53:56.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:56.315768 4084 mini_kdc.cc:253] Time spent creating user principal test-user: real 0.107s user 0.002s sys 0.003s
11:53:56.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
11:53:56.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
11:53:56.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
11:53:56.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
11:53:56.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/krb5kdc/test-user.keytab.
11:53:56.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/krb5kdc/test-user.keytab.
11:53:56.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:53:56 dist-test-slave-cvvz krb5kdc[4097](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682836, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:53:56.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
11:53:56.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T11:53:56Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
11:53:56.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T11:53:56Z Disabled control of system clock
11:53:56.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.3.253.62@KRBTEST.COM; defaulting to no policy
11:53:56.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
11:53:56.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.3.253.62@KRBTEST.COM" created.
11:53:56.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
11:53:56.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.3.253.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/krb5kdc/kudu_127.3.253.62.keytab.
11:53:56.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.3.253.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/krb5kdc/kudu_127.3.253.62.keytab.
11:53:56.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:56.502395 4084 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskrtJWFm/build/tsan/bin/kudu
11:53:56.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrtJWFm/build/tsan/bin/kudu
11:53:56.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-0/wal
11:53:56.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-0/data
11:53:56.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:53:56.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:53:56.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:53:56.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:53:56.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:53:56.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:53:56.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-0/logs
11:53:56.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-0/data/info.pb
11:53:56.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:53:56.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:53:56.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:53:56.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:53:56.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:53:56.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:53:56.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:53:56.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:53:56.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
11:53:56.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:53:56.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:53:56.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:53:56.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.253.62:46169
11:53:56.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.253.62
11:53:56.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:53:56.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/krb5kdc/kudu_127.3.253.62.keytab
11:53:56.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.3.253.62
11:53:56.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:53:56.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
11:53:56.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
11:53:56.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.253.20:34259
11:53:56.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:53:56.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:53:56.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:53:56.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:53:56.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.253.62:46169,127.3.253.61:46581,127.3.253.60:33837
11:53:56.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:53:56.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36819/jwks.json
11:53:56.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
11:53:56.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:56.889372 4110 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:53:56.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:56.890168 4110 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:53:56.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:56.891067 4110 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:53:56.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:56.928678 4110 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
11:53:56.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:56.928947 4110 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
11:53:56.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:56.929387 4110 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:53:56.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:56.929694 4110 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
11:53:56.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:56.929989 4110 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
11:53:56.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:56.930285 4110 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
11:53:56.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:56.930507 4110 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36819/jwks.json
11:53:56.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:56.968040 4110 master_runner.cc:386] Master server non-default flags:
11:53:56.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:53:56.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.253.20:34259
11:53:56.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:53:56.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:53:56.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-0/data
11:53:56.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-0/wal
11:53:56.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
11:53:56.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:53:56.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.253.62:46169,127.3.253.61:46581,127.3.253.60:33837
11:53:56.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/krb5kdc/kudu_127.3.253.62.keytab
11:53:56.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.3.253.62
11:53:56.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:53:56.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:53:56.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:53:56.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.253.62:46169
11:53:56.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:53:56.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:53:56.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:53:56.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36819/jwks.json
11:53:56.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:53:56.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
11:53:56.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
11:53:56.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:53:56.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:53:56.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-0/data/info.pb
11:53:56.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.253.62
11:53:56.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:53:56.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:53:56.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:53:56.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:53:56.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:53:56.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:53:56.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-0/logs
11:53:56.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:53:56.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:53:56.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:53:56.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
11:53:56.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:53:56.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:53:56.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:53:56.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 10:52:50 UTC on ccycloud.quasar-peqjre.root.comops.site
11:53:56.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679900
11:53:56.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:53:56.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:56.969583 4110 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:53:56.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:56.970961 4110 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:53:56.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:56.992177 4117 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:53:58.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:56.992551 4119 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:53:58.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:56.993984 4116 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:53:58.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:58.353513 4118 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
11:53:58.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:58.353638 4110 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
11:53:58.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:58.357847 4110 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:53:58.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:58.360489 4110 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:53:58.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:58.361904 4110 hybrid_clock.cc:648] HybridClock initialized: now 1790682838361872 us; error 52 us; skew 500 ppm
11:53:58.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:58.362841 4110 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:53:58.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:53:58 dist-test-slave-cvvz krb5kdc[4097](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682838, etypes {rep=17 tkt=17 ses=17}, kudu/127.3.253.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:53:58.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:58.369592 4110 init.cc:377] Logged in from keytab as kudu/127.3.253.62@KRBTEST.COM (short username kudu)
11:53:58.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:58.374562 4110 webserver.cc:460] Webserver started at http://127.3.253.62:37991/ using document root <none> and password file <none>
11:53:58.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:58.375460 4110 fs_manager.cc:359] Metadata directory not provided
11:53:58.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:58.375639 4110 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:53:58.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:58.376019 4110 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
11:53:58.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:58.381040 4110 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-0/data/instance:
11:53:58.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9e8ed01ddce3475db4624669642fe48e"
11:53:58.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:53:58 on dist-test-slave-cvvz"
11:53:58.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:58.382221 4110 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-0/wal/instance:
11:53:58.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9e8ed01ddce3475db4624669642fe48e"
11:53:58.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:53:58 on dist-test-slave-cvvz"
11:53:58.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:58.390482 4110 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
11:53:58.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:58.396883 4130 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:53:58.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:58.398164 4110 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
11:53:58.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:58.398528 4110 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-0/data,/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-0/wal
11:53:58.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9e8ed01ddce3475db4624669642fe48e"
11:53:58.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:53:58 on dist-test-slave-cvvz"
11:53:58.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:58.398945 4110 fs_report.cc:383] FS layout report
11:53:58.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:53:58.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-0/wal
11:53:58.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-0/wal
11:53:58.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-0/data/data
11:53:58.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:53:58.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:53:58.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:53:58.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:53:58.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:53:58.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:53:58.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:53:58.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:53:58.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:53:58.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:53:58.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:53:58.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:58.447443 4110 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:53:58.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:58.454283 4110 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:53:58.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:58.454794 4110 kserver.cc:163] Server-wide thread pool size limit: 3276
11:53:58.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:58.538125 4110 rpc_server.cc:307] RPC server started. Bound to: 127.3.253.62:46169
11:53:58.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:58.538197 4181 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.3.253.62:46169 every 8 connection(s)
11:53:58.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:58.540817 4110 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-0/data/info.pb
11:53:58.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:58.547715 4182 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
11:53:58.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:58.548007 4084 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskrtJWFm/build/tsan/bin/kudu as pid 4110
11:53:58.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:58.548542 4084 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-0/wal/instance
11:53:58.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:58.563854 4182 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.253.62" port: 46169 } has no permanent_uuid. Determining permanent_uuid...
11:53:58.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.3.253.61@KRBTEST.COM; defaulting to no policy
11:53:58.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
11:53:58.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.3.253.61@KRBTEST.COM" created.
11:53:58.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:53:58 dist-test-slave-cvvz krb5kdc[4097](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682838, etypes {rep=17 tkt=17 ses=17}, kudu/127.3.253.62@KRBTEST.COM for kudu/127.3.253.62@KRBTEST.COM
11:53:58.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
11:53:58.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.3.253.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/krb5kdc/kudu_127.3.253.61.keytab.
11:53:58.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.3.253.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/krb5kdc/kudu_127.3.253.61.keytab.
11:53:58.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:58.602154 4084 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskrtJWFm/build/tsan/bin/kudu
11:53:58.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrtJWFm/build/tsan/bin/kudu
11:53:58.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-1/wal
11:53:58.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-1/data
11:53:58.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:53:58.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:53:58.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:53:58.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:53:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:53:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:53:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-1/logs
11:53:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-1/data/info.pb
11:53:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:53:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:53:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:53:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:53:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:53:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:53:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:53:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:53:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
11:53:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:53:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:53:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:53:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.253.61:46581
11:53:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.253.61
11:53:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:53:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/krb5kdc/kudu_127.3.253.61.keytab
11:53:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.3.253.61
11:53:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:53:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
11:53:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
11:53:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.253.20:34259
11:53:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:53:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:53:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:53:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:53:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.253.62:46169,127.3.253.61:46581,127.3.253.60:33837
11:53:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:53:58.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36819/jwks.json
11:53:58.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
11:53:58.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:58.611603 4182 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.253.61" port: 46581 } has no permanent_uuid. Determining permanent_uuid...
11:53:58.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:58.613776 4131 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.253.61:46581: connect: Connection refused (error 111)
11:53:58.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:58.620232 4182 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.253.61:46581: Network error: Client connection negotiation failed: client connection to 127.3.253.61:46581: connect: Connection refused (error 111)
11:53:58.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:58.669762 4182 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.253.61" port: 46581 } attempt: 1
11:53:58.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:58.673712 4182 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.253.61:46581: Network error: Client connection negotiation failed: client connection to 127.3.253.61:46581: connect: Connection refused (error 111)
11:53:58.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:58.742151 4182 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.253.61" port: 46581 } attempt: 2
11:53:58.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:58.746101 4182 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.253.61:46581: Network error: Client connection negotiation failed: client connection to 127.3.253.61:46581: connect: Connection refused (error 111)
11:53:58.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:58.837486 4182 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.253.61" port: 46581 } attempt: 3
11:53:58.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:58.841045 4182 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.253.61:46581: Network error: Client connection negotiation failed: client connection to 127.3.253.61:46581: connect: Connection refused (error 111)
11:53:58.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:58.943912 4188 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:53:58.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:58.944448 4188 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:53:58.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:58.945027 4188 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:53:58.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:58.976222 4188 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
11:53:58.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:58.976450 4188 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
11:53:58.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:58.976701 4188 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:53:58.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:58.976891 4188 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
11:53:58.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:58.977178 4188 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
11:53:58.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:58.977382 4188 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
11:53:58.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:58.977514 4188 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36819/jwks.json
11:53:58.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:58.984552 4182 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.253.61" port: 46581 } attempt: 4
11:53:58.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:58.988112 4182 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.253.61:46581: Network error: Client connection negotiation failed: client connection to 127.3.253.61:46581: connect: Connection refused (error 111)
11:53:59.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:59.014619 4188 master_runner.cc:386] Master server non-default flags:
11:53:59.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:53:59.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.253.20:34259
11:53:59.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:53:59.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:53:59.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-1/data
11:53:59.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-1/wal
11:53:59.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
11:53:59.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:53:59.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.253.62:46169,127.3.253.61:46581,127.3.253.60:33837
11:53:59.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/krb5kdc/kudu_127.3.253.61.keytab
11:53:59.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.3.253.61
11:53:59.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:53:59.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:53:59.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:53:59.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.253.61:46581
11:53:59.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:53:59.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:53:59.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:53:59.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36819/jwks.json
11:53:59.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:53:59.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
11:53:59.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
11:53:59.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:53:59.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:53:59.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-1/data/info.pb
11:53:59.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.253.61
11:53:59.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:53:59.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:53:59.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:53:59.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:53:59.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:53:59.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:53:59.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-1/logs
11:53:59.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:53:59.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:53:59.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:53:59.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
11:53:59.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:53:59.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:53:59.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:53:59.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 10:52:50 UTC on ccycloud.quasar-peqjre.root.comops.site
11:53:59.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679900
11:53:59.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:53:59.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:59.015796 4188 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:53:59.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:59.016851 4188 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:53:59.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:59.037806 4196 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:53:59.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:59.287573 4182 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.253.61" port: 46581 } attempt: 5
11:53:59.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:59.292654 4182 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.253.61:46581: Network error: Client connection negotiation failed: client connection to 127.3.253.61:46581: connect: Connection refused (error 111)
11:53:59.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:59.840123 4182 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.253.61" port: 46581 } attempt: 6
11:53:59.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:59.848435 4182 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.253.61:46581: Network error: Client connection negotiation failed: client connection to 127.3.253.61:46581: connect: Connection refused (error 111)
11:54:00.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:59.037849 4198 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:54:00.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:53:59.038726 4195 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:54:00.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:53:59.042332 4188 server_base.cc:1034] running on GCE node
11:54:00.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:00.302484 4188 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:54:00.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:00.305384 4188 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:54:00.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:00.306823 4188 hybrid_clock.cc:648] HybridClock initialized: now 1790682840306782 us; error 43 us; skew 500 ppm
11:54:00.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:00.307862 4188 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:54:00.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:54:00 dist-test-slave-cvvz krb5kdc[4097](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682840, etypes {rep=17 tkt=17 ses=17}, kudu/127.3.253.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:54:00.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:00.320566 4188 init.cc:377] Logged in from keytab as kudu/127.3.253.61@KRBTEST.COM (short username kudu)
11:54:00.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:00.326012 4188 webserver.cc:460] Webserver started at http://127.3.253.61:40643/ using document root <none> and password file <none>
11:54:00.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:00.327075 4188 fs_manager.cc:359] Metadata directory not provided
11:54:00.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:00.327320 4188 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:54:00.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:00.327832 4188 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
11:54:00.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:00.332764 4188 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-1/data/instance:
11:54:00.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1f532b79ad82484685a99d7995bf7070"
11:54:00.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:54:00 on dist-test-slave-cvvz"
11:54:00.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:00.333976 4188 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-1/wal/instance:
11:54:00.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1f532b79ad82484685a99d7995bf7070"
11:54:00.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:54:00 on dist-test-slave-cvvz"
11:54:00.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:00.342085 4188 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
11:54:00.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:00.348450 4210 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:54:00.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:00.349668 4188 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
11:54:00.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:00.350054 4188 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-1/data,/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-1/wal
11:54:00.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1f532b79ad82484685a99d7995bf7070"
11:54:00.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:54:00 on dist-test-slave-cvvz"
11:54:00.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:00.350412 4188 fs_report.cc:383] FS layout report
11:54:00.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:54:00.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-1/wal
11:54:00.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-1/wal
11:54:00.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-1/data/data
11:54:00.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:54:00.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:54:00.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:54:00.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:54:00.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:54:00.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:54:00.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:54:00.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:54:00.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:54:00.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:54:00.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:54:00.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:00.392827 4188 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:54:00.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:00.399394 4188 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:54:00.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:00.400030 4188 kserver.cc:163] Server-wide thread pool size limit: 3276
11:54:00.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:00.478370 4188 rpc_server.cc:307] RPC server started. Bound to: 127.3.253.61:46581
11:54:00.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:00.478529 4261 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.3.253.61:46581 every 8 connection(s)
11:54:00.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:00.481316 4188 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-1/data/info.pb
11:54:00.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:00.483739 4084 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskrtJWFm/build/tsan/bin/kudu as pid 4188
11:54:00.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:00.484217 4084 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-1/wal/instance
11:54:00.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:00.489283 4262 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
11:54:00.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:00.503541 4262 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.253.62" port: 46169 } has no permanent_uuid. Determining permanent_uuid...
11:54:00.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.3.253.60@KRBTEST.COM; defaulting to no policy
11:54:00.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
11:54:00.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.3.253.60@KRBTEST.COM" created.
11:54:00.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:54:00 dist-test-slave-cvvz krb5kdc[4097](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682840, etypes {rep=17 tkt=17 ses=17}, kudu/127.3.253.61@KRBTEST.COM for kudu/127.3.253.62@KRBTEST.COM
11:54:00.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
11:54:00.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.3.253.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/krb5kdc/kudu_127.3.253.60.keytab.
11:54:00.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.3.253.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/krb5kdc/kudu_127.3.253.60.keytab.
11:54:00.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:00.542448 4084 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskrtJWFm/build/tsan/bin/kudu
11:54:00.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrtJWFm/build/tsan/bin/kudu
11:54:00.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-2/wal
11:54:00.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-2/data
11:54:00.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:54:00.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:54:00.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:54:00.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:54:00.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:54:00.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:54:00.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-2/logs
11:54:00.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-2/data/info.pb
11:54:00.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:54:00.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:54:00.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:54:00.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:54:00.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:54:00.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:54:00.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:54:00.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:54:00.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
11:54:00.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:54:00.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:54:00.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:54:00.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.253.60:33837
11:54:00.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.253.60
11:54:00.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:54:00.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/krb5kdc/kudu_127.3.253.60.keytab
11:54:00.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.3.253.60
11:54:00.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:54:00.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
11:54:00.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
11:54:00.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.253.20:34259
11:54:00.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:54:00.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:54:00.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:54:00.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:54:00.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.253.62:46169,127.3.253.61:46581,127.3.253.60:33837
11:54:00.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:54:00.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36819/jwks.json
11:54:00.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
11:54:00.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:00.550925 4262 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.253.61" port: 46581 } has no permanent_uuid. Determining permanent_uuid...
11:54:00.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:54:00 dist-test-slave-cvvz krb5kdc[4097](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682840, etypes {rep=17 tkt=17 ses=17}, kudu/127.3.253.61@KRBTEST.COM for kudu/127.3.253.61@KRBTEST.COM
11:54:00.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:00.584470 4262 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.253.60" port: 33837 } has no permanent_uuid. Determining permanent_uuid...
11:54:00.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:00.587319 4212 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.253.60:33837: connect: Connection refused (error 111)
11:54:00.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:00.591971 4262 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.253.60:33837: Network error: Client connection negotiation failed: client connection to 127.3.253.60:33837: connect: Connection refused (error 111)
11:54:00.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:00.641497 4262 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.253.60" port: 33837 } attempt: 1
11:54:00.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:00.646361 4262 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.253.60:33837: Network error: Client connection negotiation failed: client connection to 127.3.253.60:33837: connect: Connection refused (error 111)
11:54:00.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:00.714784 4262 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.253.60" port: 33837 } attempt: 2
11:54:00.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:00.719907 4262 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.253.60:33837: Network error: Client connection negotiation failed: client connection to 127.3.253.60:33837: connect: Connection refused (error 111)
11:54:00.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:00.811478 4262 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.253.60" port: 33837 } attempt: 3
11:54:00.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:00.815469 4262 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.253.60:33837: Network error: Client connection negotiation failed: client connection to 127.3.253.60:33837: connect: Connection refused (error 111)
11:54:00.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:00.893855 4268 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:54:00.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:00.894706 4268 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:54:00.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:00.895866 4268 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:54:00.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:00.908900 4182 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.253.61" port: 46581 } attempt: 7
11:54:00.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:54:00 dist-test-slave-cvvz krb5kdc[4097](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682838, etypes {rep=17 tkt=17 ses=17}, kudu/127.3.253.62@KRBTEST.COM for kudu/127.3.253.61@KRBTEST.COM
11:54:00.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:00.937671 4268 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
11:54:00.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:00.937677 4182 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.253.60" port: 33837 } has no permanent_uuid. Determining permanent_uuid...
11:54:00.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:00.937954 4268 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
11:54:00.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:00.938374 4268 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:54:00.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:00.938732 4268 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
11:54:00.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:00.939087 4268 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
11:54:00.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:00.939481 4268 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
11:54:00.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:00.939682 4268 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36819/jwks.json
11:54:00.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:00.942978 4182 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.253.60:33837: Network error: Client connection negotiation failed: client connection to 127.3.253.60:33837: connect: Connection refused (error 111)
11:54:00.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:00.958911 4262 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.253.60" port: 33837 } attempt: 4
11:54:00.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:00.962990 4262 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.253.60:33837: Network error: Client connection negotiation failed: client connection to 127.3.253.60:33837: connect: Connection refused (error 111)
11:54:00.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:00.982182 4268 master_runner.cc:386] Master server non-default flags:
11:54:00.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:54:00.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.253.20:34259
11:54:00.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:54:00.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:54:00.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-2/data
11:54:00.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-2/wal
11:54:00.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
11:54:00.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:54:00.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.253.62:46169,127.3.253.61:46581,127.3.253.60:33837
11:54:00.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/krb5kdc/kudu_127.3.253.60.keytab
11:54:00.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.3.253.60
11:54:00.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:54:00.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:54:00.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:54:00.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.253.60:33837
11:54:00.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:54:00.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:54:00.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:54:00.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36819/jwks.json
11:54:00.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:54:00.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
11:54:00.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
11:54:00.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:54:00.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:54:00.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-2/data/info.pb
11:54:00.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.253.60
11:54:00.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:54:00.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:54:00.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:54:00.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:54:00.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:54:00.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:54:00.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-2/logs
11:54:00.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:54:00.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:54:00.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:54:00.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
11:54:00.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:54:00.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:54:00.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:54:00.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 10:52:50 UTC on ccycloud.quasar-peqjre.root.comops.site
11:54:00.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679900
11:54:00.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:54:00.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:00.983376 4268 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:54:00.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:00.984375 4268 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:54:01.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:01.001353 4182 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.253.60" port: 33837 } attempt: 1
11:54:01.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:01.004297 4280 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:54:01.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:01.006479 4182 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.253.60:33837: Network error: Client connection negotiation failed: client connection to 127.3.253.60:33837: connect: Connection refused (error 111)
11:54:01.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:01.087980 4182 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.253.60" port: 33837 } attempt: 2
11:54:01.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:01.092308 4182 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.253.60:33837: Network error: Client connection negotiation failed: client connection to 127.3.253.60:33837: connect: Connection refused (error 111)
11:54:01.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:01.178004 4182 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.253.60" port: 33837 } attempt: 3
11:54:01.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:01.182279 4182 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.253.60:33837: Network error: Client connection negotiation failed: client connection to 127.3.253.60:33837: connect: Connection refused (error 111)
11:54:01.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:01.262707 4262 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.253.60" port: 33837 } attempt: 5
11:54:01.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:01.266855 4262 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.253.60:33837: Network error: Client connection negotiation failed: client connection to 127.3.253.60:33837: connect: Connection refused (error 111)
11:54:01.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:01.322867 4182 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.253.60" port: 33837 } attempt: 4
11:54:01.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:01.326534 4182 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.253.60:33837: Network error: Client connection negotiation failed: client connection to 127.3.253.60:33837: connect: Connection refused (error 111)
11:54:01.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:01.610138 4182 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.253.60" port: 33837 } attempt: 5
11:54:01.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:01.614185 4182 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.253.60:33837: Network error: Client connection negotiation failed: client connection to 127.3.253.60:33837: connect: Connection refused (error 111)
11:54:01.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:01.814268 4262 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.253.60" port: 33837 } attempt: 6
11:54:01.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:01.822824 4262 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.253.60:33837: Network error: Client connection negotiation failed: client connection to 127.3.253.60:33837: connect: Connection refused (error 111)
11:54:02.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:02.166559 4182 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.253.60" port: 33837 } attempt: 6
11:54:02.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:02.173480 4182 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.253.60:33837: Network error: Client connection negotiation failed: client connection to 127.3.253.60:33837: connect: Connection refused (error 111)
11:54:02.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:01.005716 4279 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:54:02.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:01.007335 4282 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:54:02.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:02.308846 4281 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
11:54:02.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:02.308887 4268 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
11:54:02.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:02.314334 4268 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:54:02.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:02.317692 4268 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:54:02.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:02.319243 4268 hybrid_clock.cc:648] HybridClock initialized: now 1790682842319172 us; error 73 us; skew 500 ppm
11:54:02.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:02.320238 4268 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:54:02.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:54:02 dist-test-slave-cvvz krb5kdc[4097](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682842, etypes {rep=17 tkt=17 ses=17}, kudu/127.3.253.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:54:02.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:02.327346 4268 init.cc:377] Logged in from keytab as kudu/127.3.253.60@KRBTEST.COM (short username kudu)
11:54:02.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:02.333372 4268 webserver.cc:460] Webserver started at http://127.3.253.60:41543/ using document root <none> and password file <none>
11:54:02.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:02.334342 4268 fs_manager.cc:359] Metadata directory not provided
11:54:02.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:02.334528 4268 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:54:02.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:02.334924 4268 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
11:54:02.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:02.339838 4268 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-2/data/instance:
11:54:02.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "28f41463e82d4a36b1ff698a18ec239d"
11:54:02.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:54:02 on dist-test-slave-cvvz"
11:54:02.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:02.340981 4268 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-2/wal/instance:
11:54:02.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "28f41463e82d4a36b1ff698a18ec239d"
11:54:02.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:54:02 on dist-test-slave-cvvz"
11:54:02.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:02.349174 4268 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
11:54:02.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:02.355602 4296 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:54:02.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:02.356896 4268 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.004s
11:54:02.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:02.357391 4268 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-2/data,/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-2/wal
11:54:02.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "28f41463e82d4a36b1ff698a18ec239d"
11:54:02.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:54:02 on dist-test-slave-cvvz"
11:54:02.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:02.357723 4268 fs_report.cc:383] FS layout report
11:54:02.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:54:02.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-2/wal
11:54:02.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-2/wal
11:54:02.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-2/data/data
11:54:02.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:54:02.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:54:02.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:54:02.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:54:02.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:54:02.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:54:02.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:54:02.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:54:02.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:54:02.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:54:02.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:54:02.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:02.408872 4268 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:54:02.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:02.417366 4268 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:54:02.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:02.418059 4268 kserver.cc:163] Server-wide thread pool size limit: 3276
11:54:02.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:02.515151 4268 rpc_server.cc:307] RPC server started. Bound to: 127.3.253.60:33837
11:54:02.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:02.515220 4347 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.3.253.60:33837 every 8 connection(s)
11:54:02.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:02.518137 4268 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-2/data/info.pb
11:54:02.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:02.519289 4084 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskrtJWFm/build/tsan/bin/kudu as pid 4268
11:54:02.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:02.519757 4084 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-2/wal/instance
11:54:02.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:02.526893 4348 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
11:54:02.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:02.548568 4348 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.253.62" port: 46169 } has no permanent_uuid. Determining permanent_uuid...
11:54:02.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:54:02 dist-test-slave-cvvz krb5kdc[4097](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682836, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.3.253.60@KRBTEST.COM
11:54:02.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:54:02 dist-test-slave-cvvz krb5kdc[4097](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682842, etypes {rep=17 tkt=17 ses=17}, kudu/127.3.253.60@KRBTEST.COM for kudu/127.3.253.62@KRBTEST.COM
11:54:02.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:02.591564 4348 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.253.61" port: 46581 } has no permanent_uuid. Determining permanent_uuid...
11:54:02.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:54:02 dist-test-slave-cvvz krb5kdc[4097](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682836, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.3.253.61@KRBTEST.COM
11:54:02.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:54:02 dist-test-slave-cvvz krb5kdc[4097](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682842, etypes {rep=17 tkt=17 ses=17}, kudu/127.3.253.60@KRBTEST.COM for kudu/127.3.253.61@KRBTEST.COM
11:54:02.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:02.630740 4348 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.253.60" port: 33837 } has no permanent_uuid. Determining permanent_uuid...
11:54:02.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:54:02 dist-test-slave-cvvz krb5kdc[4097](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682842, etypes {rep=17 tkt=17 ses=17}, kudu/127.3.253.60@KRBTEST.COM for kudu/127.3.253.60@KRBTEST.COM
11:54:02.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:54:02 dist-test-slave-cvvz krb5kdc[4097](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682836, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.3.253.62@KRBTEST.COM
11:54:02.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:02.672892 4348 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d: Bootstrap starting.
11:54:02.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:02.686348 4348 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d: Neither blocks nor log segments found. Creating new log.
11:54:02.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:02.688822 4348 log.cc:824] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d: Log is configured to *not* fsync() on all Append() calls
11:54:02.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:02.692459 4084 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
11:54:02.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:02.696139 4348 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d: No bootstrap required, opened a new log
11:54:02.708 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:54:02.709 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
11:54:02.723 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
11:54:02.725 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
11:54:02.728 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.3.253.62"
port: 46169
}
bound_http_address {
host: "127.3.253.62"
port: 37991
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.3.253.61"
port: 46581
}
bound_http_address {
host: "127.3.253.61"
port: 40643
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.3.253.60"
port: 33837
}
bound_http_address {
host: "127.3.253.60"
port: 41543
}
}
}
11:54:02.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:02.727771 4348 raft_consensus.cc:348] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8ed01ddce3475db4624669642fe48e" member_type: VOTER last_known_addr { host: "127.3.253.62" port: 46169 } } peers { permanent_uuid: "1f532b79ad82484685a99d7995bf7070" member_type: VOTER last_known_addr { host: "127.3.253.61" port: 46581 } } peers { permanent_uuid: "28f41463e82d4a36b1ff698a18ec239d" member_type: VOTER last_known_addr { host: "127.3.253.60" port: 33837 } }
11:54:02.729 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
11:54:02.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:02.728875 4348 raft_consensus.cc:374] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
11:54:02.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:02.729297 4348 raft_consensus.cc:729] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 28f41463e82d4a36b1ff698a18ec239d, State: Initialized, Role: FOLLOWER
11:54:02.729 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:54:02.729 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
11:54:02.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:02.730255 4348 consensus_queue.cc:260] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8ed01ddce3475db4624669642fe48e" member_type: VOTER last_known_addr { host: "127.3.253.62" port: 46169 } } peers { permanent_uuid: "1f532b79ad82484685a99d7995bf7070" member_type: VOTER last_known_addr { host: "127.3.253.61" port: 46581 } } peers { permanent_uuid: "28f41463e82d4a36b1ff698a18ec239d" member_type: VOTER last_known_addr { host: "127.3.253.60" port: 33837 } }
11:54:02.731 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
11:54:02.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:02.733547 4358 sys_catalog.cc:455] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8ed01ddce3475db4624669642fe48e" member_type: VOTER last_known_addr { host: "127.3.253.62" port: 46169 } } peers { permanent_uuid: "1f532b79ad82484685a99d7995bf7070" member_type: VOTER last_known_addr { host: "127.3.253.61" port: 46581 } } peers { permanent_uuid: "28f41463e82d4a36b1ff698a18ec239d" member_type: VOTER last_known_addr { host: "127.3.253.60" port: 33837 } } }
11:54:02.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:02.734633 4358 sys_catalog.cc:458] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d [sys.catalog]: This master's current role is: FOLLOWER
11:54:02.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:02.735752 4348 sys_catalog.cc:564] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d [sys.catalog]: configured and running, proceeding with master startup.
11:54:02.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:54:02 dist-test-slave-cvvz krb5kdc[4097](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682842, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:54:02.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
11:54:02.768 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:54:02.770 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/krb5kdc/krb5cc
11:54:02.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:02.772275 4371 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:54:02.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:02.772639 4371 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:54:02.780 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
11:54:02.787 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/krb5kdc/krb5cc
11:54:02.793 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
11:54:02.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:02.762984 4268 master_runner.cc:186] Error getting master registration for 127.3.253.62:46169: OK, instance_id { permanent_uuid: "9e8ed01ddce3475db4624669642fe48e" instance_seqno: 1790682838517417 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
11:54:02.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:02.797326 4268 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
11:54:02.797 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
11:54:02.799 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
11:54:02.800 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
11:54:02.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:54:02 dist-test-slave-cvvz krb5kdc[4097](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682842, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:54:02.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
11:54:02.842 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:54:02.842 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/krb5kdc/krb5cc
11:54:02.844 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
11:54:02.846 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.3.253.60:33837
11:54:02.848 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
11:54:02.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:02.849404 4084 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskrtJWFm/build/tsan/bin/kudu with pid 4268
11:54:02.871 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:54:02.871 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.3.253.62:46169
11:54:02.872 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
11:54:02.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:02.872368 4084 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskrtJWFm/build/tsan/bin/kudu with pid 4110
11:54:02.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:02.883543 4262 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.253.60" port: 33837 } attempt: 7
11:54:02.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:02.890969 4262 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.253.60:33837: Network error: Client connection negotiation failed: client connection to 127.3.253.60:33837: connect: Connection refused (error 111)
11:54:02.899 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:54:02.899 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.3.253.61:46581
11:54:02.899 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
11:54:02.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:02.900218 4084 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskrtJWFm/build/tsan/bin/kudu with pid 4188
11:54:02.923 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:54:02.923 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.3.253.60:33837
11:54:02.925 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
11:54:02.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:02.928411 4084 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskrtJWFm/build/tsan/bin/kudu
11:54:02.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrtJWFm/build/tsan/bin/kudu
11:54:02.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-2/wal
11:54:02.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-2/data
11:54:02.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:54:02.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:54:02.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:54:02.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:54:02.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:54:02.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:54:02.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-2/logs
11:54:02.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-2/data/info.pb
11:54:02.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:54:02.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:54:02.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:54:02.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:54:02.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:54:02.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:54:02.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:54:02.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:54:02.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
11:54:02.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:54:02.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:54:02.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:54:02.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.253.60:33837
11:54:02.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.253.60
11:54:02.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41543
11:54:02.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/krb5kdc/kudu_127.3.253.60.keytab
11:54:02.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.3.253.60
11:54:02.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:54:02.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
11:54:02.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
11:54:02.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.253.20:34259
11:54:02.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:54:02.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:54:02.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:54:02.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:54:02.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.253.62:46169,127.3.253.61:46581,127.3.253.60:33837
11:54:02.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:54:02.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36819/jwks.json
11:54:02.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
11:54:03.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:03.259480 4377 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:54:03.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:03.260082 4377 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:54:03.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:03.260710 4377 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:54:03.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:03.294677 4377 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
11:54:03.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:03.294926 4377 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
11:54:03.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:03.295186 4377 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:54:03.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:03.295387 4377 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
11:54:03.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:03.295588 4377 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
11:54:03.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:03.295773 4377 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
11:54:03.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:03.295919 4377 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36819/jwks.json
11:54:03.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:03.335131 4377 master_runner.cc:386] Master server non-default flags:
11:54:03.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:54:03.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.253.20:34259
11:54:03.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:54:03.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:54:03.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-2/data
11:54:03.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-2/wal
11:54:03.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
11:54:03.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:54:03.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.253.62:46169,127.3.253.61:46581,127.3.253.60:33837
11:54:03.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/krb5kdc/kudu_127.3.253.60.keytab
11:54:03.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.3.253.60
11:54:03.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:54:03.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:54:03.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:54:03.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.253.60:33837
11:54:03.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:54:03.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:54:03.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:54:03.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36819/jwks.json
11:54:03.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:54:03.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
11:54:03.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
11:54:03.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:54:03.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:54:03.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-2/data/info.pb
11:54:03.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.253.60
11:54:03.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41543
11:54:03.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:54:03.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:54:03.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:54:03.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:54:03.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:54:03.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-2/logs
11:54:03.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:54:03.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:54:03.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:54:03.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
11:54:03.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:54:03.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:54:03.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:54:03.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 10:52:50 UTC on ccycloud.quasar-peqjre.root.comops.site
11:54:03.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679900
11:54:03.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:54:03.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:03.336695 4377 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:54:03.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:03.337864 4377 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:54:03.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:03.355288 4383 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:54:03.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:03.693459 4068 debug-util.cc:398] Leaking SignalData structure 0x7b08000af940 after lost signal to thread 3944
11:54:03.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:03.694626 4068 debug-util.cc:398] Leaking SignalData structure 0x7b08000af680 after lost signal to thread 4071
11:54:04.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:03.355358 4384 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:54:04.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:03.355722 4386 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:54:04.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:03.357246 4377 server_base.cc:1034] running on GCE node
11:54:04.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:04.735430 4377 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:54:04.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:04.739292 4377 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:54:04.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:04.740753 4377 hybrid_clock.cc:648] HybridClock initialized: now 1790682844740729 us; error 43 us; skew 500 ppm
11:54:04.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:04.741726 4377 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:54:04.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:54:04 dist-test-slave-cvvz krb5kdc[4097](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682844, etypes {rep=17 tkt=17 ses=17}, kudu/127.3.253.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:54:04.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:04.748056 4377 init.cc:377] Logged in from keytab as kudu/127.3.253.60@KRBTEST.COM (short username kudu)
11:54:04.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:04.754323 4377 webserver.cc:460] Webserver started at http://127.3.253.60:41543/ using document root <none> and password file <none>
11:54:04.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:04.755498 4377 fs_manager.cc:359] Metadata directory not provided
11:54:04.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:04.755759 4377 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:54:04.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:04.765633 4377 fs_manager.cc:709] Time spent opening directory manager: real 0.007s user 0.006s sys 0.001s
11:54:04.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:04.771740 4394 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:54:04.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:04.773223 4377 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.006s sys 0.001s
11:54:04.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:04.773684 4377 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-2/data,/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-2/wal
11:54:04.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "28f41463e82d4a36b1ff698a18ec239d"
11:54:04.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:54:02 on dist-test-slave-cvvz"
11:54:04.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:04.777412 4377 fs_report.cc:383] FS layout report
11:54:04.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:54:04.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-2/wal
11:54:04.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-2/wal
11:54:04.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-2/data/data
11:54:04.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:54:04.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:54:04.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:54:04.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:54:04.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:54:04.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:54:04.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:54:04.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:54:04.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:54:04.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:54:04.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:54:04.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:04.831799 4377 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:54:04.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:04.840601 4377 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:54:04.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:04.841255 4377 kserver.cc:163] Server-wide thread pool size limit: 3276
11:54:04.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:04.941751 4377 rpc_server.cc:307] RPC server started. Bound to: 127.3.253.60:33837
11:54:04.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:04.941864 4446 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.3.253.60:33837 every 8 connection(s)
11:54:04.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:04.945870 4377 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-2/data/info.pb
11:54:04.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:04.949685 4084 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskrtJWFm/build/tsan/bin/kudu as pid 4377
11:54:04.950 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:54:04.950 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.3.253.62:46169
11:54:04.951 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
11:54:04.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:04.952930 4084 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskrtJWFm/build/tsan/bin/kudu
11:54:04.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrtJWFm/build/tsan/bin/kudu
11:54:04.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-0/wal
11:54:04.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-0/data
11:54:04.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:54:04.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:54:04.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:54:04.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:54:04.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:54:04.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:54:04.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-0/logs
11:54:04.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-0/data/info.pb
11:54:04.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:54:04.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:54:04.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:54:04.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:54:04.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:54:04.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:54:04.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:54:04.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:54:04.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
11:54:04.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:54:04.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:54:04.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:54:04.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.253.62:46169
11:54:04.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.253.62
11:54:04.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37991
11:54:04.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/krb5kdc/kudu_127.3.253.62.keytab
11:54:04.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.3.253.62
11:54:04.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:54:04.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
11:54:04.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
11:54:04.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.253.20:34259
11:54:04.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:54:04.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:54:04.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:54:04.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:54:04.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.253.62:46169,127.3.253.61:46581,127.3.253.60:33837
11:54:04.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:54:04.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36819/jwks.json
11:54:04.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
11:54:04.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:04.960016 4447 sys_catalog.cc:263] Verifying existing consensus state
11:54:04.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:04.970423 4447 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d: Bootstrap starting.
11:54:05.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:05.015187 4447 log.cc:824] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d: Log is configured to *not* fsync() on all Append() calls
11:54:05.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:05.020865 4447 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
11:54:05.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:05.021766 4447 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d: Bootstrap complete.
11:54:05.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:05.058116 4447 raft_consensus.cc:348] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8ed01ddce3475db4624669642fe48e" member_type: VOTER last_known_addr { host: "127.3.253.62" port: 46169 } } peers { permanent_uuid: "1f532b79ad82484685a99d7995bf7070" member_type: VOTER last_known_addr { host: "127.3.253.61" port: 46581 } } peers { permanent_uuid: "28f41463e82d4a36b1ff698a18ec239d" member_type: VOTER last_known_addr { host: "127.3.253.60" port: 33837 } }
11:54:05.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:05.061597 4447 raft_consensus.cc:374] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
11:54:05.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:05.061887 4447 raft_consensus.cc:729] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 28f41463e82d4a36b1ff698a18ec239d, State: Initialized, Role: FOLLOWER
11:54:05.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:05.062805 4447 consensus_queue.cc:260] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8ed01ddce3475db4624669642fe48e" member_type: VOTER last_known_addr { host: "127.3.253.62" port: 46169 } } peers { permanent_uuid: "1f532b79ad82484685a99d7995bf7070" member_type: VOTER last_known_addr { host: "127.3.253.61" port: 46581 } } peers { permanent_uuid: "28f41463e82d4a36b1ff698a18ec239d" member_type: VOTER last_known_addr { host: "127.3.253.60" port: 33837 } }
11:54:05.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:05.066306 4451 sys_catalog.cc:455] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8ed01ddce3475db4624669642fe48e" member_type: VOTER last_known_addr { host: "127.3.253.62" port: 46169 } } peers { permanent_uuid: "1f532b79ad82484685a99d7995bf7070" member_type: VOTER last_known_addr { host: "127.3.253.61" port: 46581 } } peers { permanent_uuid: "28f41463e82d4a36b1ff698a18ec239d" member_type: VOTER last_known_addr { host: "127.3.253.60" port: 33837 } } }
11:54:05.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:05.067579 4451 sys_catalog.cc:458] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d [sys.catalog]: This master's current role is: FOLLOWER
11:54:05.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:05.067709 4447 sys_catalog.cc:564] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d [sys.catalog]: configured and running, proceeding with master startup.
11:54:05.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:05.098093 4400 tablet.cc:2369] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d: Can't schedule compaction. Clean time has not been advanced past its initial value.
11:54:05.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:05.101118 4451 raft_consensus.cc:482] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
11:54:05.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:05.101676 4451 raft_consensus.cc:504] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8ed01ddce3475db4624669642fe48e" member_type: VOTER last_known_addr { host: "127.3.253.62" port: 46169 } } peers { permanent_uuid: "1f532b79ad82484685a99d7995bf7070" member_type: VOTER last_known_addr { host: "127.3.253.61" port: 46581 } } peers { permanent_uuid: "28f41463e82d4a36b1ff698a18ec239d" member_type: VOTER last_known_addr { host: "127.3.253.60" port: 33837 } }
11:54:05.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:05.106202 4451 leader_election.cc:288] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 9e8ed01ddce3475db4624669642fe48e (127.3.253.62:46169), 1f532b79ad82484685a99d7995bf7070 (127.3.253.61:46581)
11:54:05.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:05.122206 4395 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.253.62:46169: connect: Connection refused (error 111)
11:54:05.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:05.128449 4462 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:54:05.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:05.128809 4462 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:54:05.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:05.154160 4395 leader_election.cc:334] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 9e8ed01ddce3475db4624669642fe48e (127.3.253.62:46169): Network error: Client connection negotiation failed: client connection to 127.3.253.62:46169: connect: Connection refused (error 111)
11:54:05.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:05.168570 4395 leader_election.cc:334] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 1f532b79ad82484685a99d7995bf7070 (127.3.253.61:46581): Network error: Client connection negotiation failed: client connection to 127.3.253.61:46581: connect: Connection refused (error 111)
11:54:05.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:05.170827 4395 leader_election.cc:302] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 28f41463e82d4a36b1ff698a18ec239d; no voters: 1f532b79ad82484685a99d7995bf7070, 9e8ed01ddce3475db4624669642fe48e
11:54:05.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:05.173223 4451 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
11:54:05.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:05.136762 4377 master_runner.cc:186] Error getting master registration for 127.3.253.62:46169: Network error: Client connection negotiation failed: client connection to 127.3.253.62:46169: connect: Connection refused (error 111),
11:54:05.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:05.200784 4377 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
11:54:05.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:05.486083 4449 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:54:05.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:05.486860 4449 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:54:05.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:05.487884 4449 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:54:05.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:05.532214 4449 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
11:54:05.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:05.532496 4449 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
11:54:05.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:05.532888 4449 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:54:05.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:05.533246 4449 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
11:54:05.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:05.533555 4449 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
11:54:05.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:05.533879 4449 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
11:54:05.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:05.534116 4449 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36819/jwks.json
11:54:05.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:05.574018 4449 master_runner.cc:386] Master server non-default flags:
11:54:05.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:54:05.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.253.20:34259
11:54:05.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:54:05.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:54:05.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-0/data
11:54:05.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-0/wal
11:54:05.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
11:54:05.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:54:05.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.253.62:46169,127.3.253.61:46581,127.3.253.60:33837
11:54:05.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/krb5kdc/kudu_127.3.253.62.keytab
11:54:05.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.3.253.62
11:54:05.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:54:05.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:54:05.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:54:05.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.253.62:46169
11:54:05.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:54:05.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:54:05.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:54:05.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36819/jwks.json
11:54:05.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:54:05.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
11:54:05.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
11:54:05.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:54:05.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:54:05.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-0/data/info.pb
11:54:05.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.253.62
11:54:05.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37991
11:54:05.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:54:05.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:54:05.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:54:05.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:54:05.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:54:05.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-0/logs
11:54:05.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:54:05.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:54:05.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:54:05.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
11:54:05.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:54:05.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:54:05.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:54:05.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 10:52:50 UTC on ccycloud.quasar-peqjre.root.comops.site
11:54:05.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679900
11:54:05.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:54:05.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:05.575569 4449 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:54:05.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:05.576987 4449 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:54:05.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:05.601655 4472 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:54:06.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:06.134140 4462 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:54:06.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:06.134960 4462 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:54:07.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:07.139879 4462 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:54:07.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:07.141403 4462 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:54:07.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:07.254012 4478 raft_consensus.cc:482] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
11:54:07.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:07.255079 4478 raft_consensus.cc:504] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8ed01ddce3475db4624669642fe48e" member_type: VOTER last_known_addr { host: "127.3.253.62" port: 46169 } } peers { permanent_uuid: "1f532b79ad82484685a99d7995bf7070" member_type: VOTER last_known_addr { host: "127.3.253.61" port: 46581 } } peers { permanent_uuid: "28f41463e82d4a36b1ff698a18ec239d" member_type: VOTER last_known_addr { host: "127.3.253.60" port: 33837 } }
11:54:07.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:07.265569 4478 leader_election.cc:288] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 9e8ed01ddce3475db4624669642fe48e (127.3.253.62:46169), 1f532b79ad82484685a99d7995bf7070 (127.3.253.61:46581)
11:54:07.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:07.290258 4395 leader_election.cc:334] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 9e8ed01ddce3475db4624669642fe48e (127.3.253.62:46169): Network error: Client connection negotiation failed: client connection to 127.3.253.62:46169: connect: Connection refused (error 111)
11:54:07.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:07.293012 4395 leader_election.cc:334] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 1f532b79ad82484685a99d7995bf7070 (127.3.253.61:46581): Network error: Client connection negotiation failed: client connection to 127.3.253.61:46581: connect: Connection refused (error 111)
11:54:07.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:07.294227 4395 leader_election.cc:302] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 28f41463e82d4a36b1ff698a18ec239d; no voters: 1f532b79ad82484685a99d7995bf7070, 9e8ed01ddce3475db4624669642fe48e
11:54:07.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:07.295682 4478 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
11:54:07.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:06.996625 4471 debug-util.cc:398] Leaking SignalData structure 0x7b0800014920 after lost signal to thread 4449
11:54:07.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:07.705905 4449 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 2.105s user 0.658s sys 1.092s
11:54:07.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:07.710948 4449 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 2.110s user 0.658s sys 1.092s
11:54:07.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:05.602694 4473 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:54:07.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:07.709337 4475 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:54:07.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:07.716398 4474 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2105 milliseconds
11:54:07.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:07.717134 4449 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
11:54:07.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:07.719051 4449 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:54:07.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:07.728485 4449 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:54:07.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:07.729930 4449 hybrid_clock.cc:648] HybridClock initialized: now 1790682847729893 us; error 56 us; skew 500 ppm
11:54:07.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:07.730942 4449 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:54:07.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:54:07 dist-test-slave-cvvz krb5kdc[4097](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682847, etypes {rep=17 tkt=17 ses=17}, kudu/127.3.253.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:54:07.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:07.742316 4449 init.cc:377] Logged in from keytab as kudu/127.3.253.62@KRBTEST.COM (short username kudu)
11:54:07.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:07.748878 4449 webserver.cc:460] Webserver started at http://127.3.253.62:37991/ using document root <none> and password file <none>
11:54:07.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:07.749897 4449 fs_manager.cc:359] Metadata directory not provided
11:54:07.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:07.750118 4449 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:54:07.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:07.760496 4449 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.006s sys 0.000s
11:54:07.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:07.765703 4489 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:54:07.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:07.767125 4449 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.000s sys 0.004s
11:54:07.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:07.767459 4449 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-0/data,/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-0/wal
11:54:07.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9e8ed01ddce3475db4624669642fe48e"
11:54:07.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:53:58 on dist-test-slave-cvvz"
11:54:07.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:07.770078 4449 fs_report.cc:383] FS layout report
11:54:07.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:54:07.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-0/wal
11:54:07.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-0/wal
11:54:07.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-0/data/data
11:54:07.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:54:07.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:54:07.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:54:07.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:54:07.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:54:07.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:54:07.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:54:07.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:54:07.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:54:07.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:54:07.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:54:07.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:07.808768 4449 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:54:07.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:07.815234 4449 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:54:07.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:07.815791 4449 kserver.cc:163] Server-wide thread pool size limit: 3276
11:54:07.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:07.910429 4449 rpc_server.cc:307] RPC server started. Bound to: 127.3.253.62:46169
11:54:07.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:07.910583 4540 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.3.253.62:46169 every 8 connection(s)
11:54:07.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:07.913971 4449 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-0/data/info.pb
11:54:07.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:07.915964 4084 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskrtJWFm/build/tsan/bin/kudu as pid 4449
11:54:07.918 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:54:07.918 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.3.253.61:46581
11:54:07.918 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
11:54:07.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:07.920392 4084 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskrtJWFm/build/tsan/bin/kudu
11:54:07.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrtJWFm/build/tsan/bin/kudu
11:54:07.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-1/wal
11:54:07.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-1/data
11:54:07.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:54:07.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:54:07.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:54:07.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:54:07.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:54:07.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:54:07.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-1/logs
11:54:07.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-1/data/info.pb
11:54:07.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:54:07.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:54:07.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:54:07.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:54:07.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:54:07.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:54:07.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:54:07.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:54:07.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
11:54:07.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:54:07.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:54:07.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:54:07.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.253.61:46581
11:54:07.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.253.61
11:54:07.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40643
11:54:07.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/krb5kdc/kudu_127.3.253.61.keytab
11:54:07.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.3.253.61
11:54:07.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:54:07.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
11:54:07.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
11:54:07.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.253.20:34259
11:54:07.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:54:07.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:54:07.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:54:07.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:54:07.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.253.62:46169,127.3.253.61:46581,127.3.253.60:33837
11:54:07.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:54:07.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36819/jwks.json
11:54:07.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
11:54:07.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:07.934810 4541 sys_catalog.cc:263] Verifying existing consensus state
11:54:07.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260929 11:54:07.935755 4541 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
11:54:07.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:07.936851 4449 master.cc:537] Master@127.3.253.62:46169 shutting down...
11:54:08.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:08.019272 4449 master.cc:552] Master@127.3.253.62:46169 shutdown complete.
11:54:08.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
11:54:08.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:08.147207 4462 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:54:08.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:08.147969 4462 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:54:08.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:08.338816 4543 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:54:08.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:08.339537 4543 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:54:08.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:08.340353 4543 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:54:08.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:08.376019 4543 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
11:54:08.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:08.376320 4543 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
11:54:08.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:08.376739 4543 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:54:08.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:08.377061 4543 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
11:54:08.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:08.377421 4543 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
11:54:08.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:08.377722 4543 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
11:54:08.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:08.377935 4543 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36819/jwks.json
11:54:08.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:08.420485 4543 master_runner.cc:386] Master server non-default flags:
11:54:08.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:54:08.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.253.20:34259
11:54:08.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:54:08.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:54:08.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-1/data
11:54:08.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-1/wal
11:54:08.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
11:54:08.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:54:08.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.253.62:46169,127.3.253.61:46581,127.3.253.60:33837
11:54:08.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/krb5kdc/kudu_127.3.253.61.keytab
11:54:08.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.3.253.61
11:54:08.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:54:08.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:54:08.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:54:08.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.253.61:46581
11:54:08.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:54:08.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:54:08.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:54:08.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36819/jwks.json
11:54:08.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:54:08.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
11:54:08.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
11:54:08.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:54:08.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:54:08.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-1/data/info.pb
11:54:08.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.253.61
11:54:08.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40643
11:54:08.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:54:08.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:54:08.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:54:08.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:54:08.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:54:08.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-1/logs
11:54:08.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:54:08.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:54:08.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:54:08.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
11:54:08.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:54:08.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:54:08.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:54:08.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 10:52:50 UTC on ccycloud.quasar-peqjre.root.comops.site
11:54:08.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679900
11:54:08.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:54:08.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:08.422065 4543 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:54:08.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:08.423442 4543 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:54:08.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:08.448364 4551 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:54:09.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:09.157568 4462 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:54:09.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:09.158635 4462 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:54:09.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:09.198823 4557 raft_consensus.cc:482] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
11:54:09.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:09.200644 4557 raft_consensus.cc:504] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8ed01ddce3475db4624669642fe48e" member_type: VOTER last_known_addr { host: "127.3.253.62" port: 46169 } } peers { permanent_uuid: "1f532b79ad82484685a99d7995bf7070" member_type: VOTER last_known_addr { host: "127.3.253.61" port: 46581 } } peers { permanent_uuid: "28f41463e82d4a36b1ff698a18ec239d" member_type: VOTER last_known_addr { host: "127.3.253.60" port: 33837 } }
11:54:09.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:09.213523 4557 leader_election.cc:288] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 9e8ed01ddce3475db4624669642fe48e (127.3.253.62:46169), 1f532b79ad82484685a99d7995bf7070 (127.3.253.61:46581)
11:54:09.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:09.234141 4395 leader_election.cc:334] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 9e8ed01ddce3475db4624669642fe48e (127.3.253.62:46169): Network error: Client connection negotiation failed: client connection to 127.3.253.62:46169: connect: Connection refused (error 111)
11:54:09.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:09.243046 4395 leader_election.cc:334] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 1f532b79ad82484685a99d7995bf7070 (127.3.253.61:46581): Network error: Client connection negotiation failed: client connection to 127.3.253.61:46581: connect: Connection refused (error 111)
11:54:09.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:09.244459 4395 leader_election.cc:302] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 28f41463e82d4a36b1ff698a18ec239d; no voters: 1f532b79ad82484685a99d7995bf7070, 9e8ed01ddce3475db4624669642fe48e
11:54:09.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:09.246352 4557 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
11:54:10.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:09.844676 4550 debug-util.cc:398] Leaking SignalData structure 0x7b0800014920 after lost signal to thread 4543
11:54:10.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:10.020071 4543 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 1.572s user 0.487s sys 1.076s
11:54:10.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:10.020533 4543 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 1.573s user 0.487s sys 1.076s
11:54:10.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:08.448622 4552 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:54:10.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:10.023258 4554 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:54:10.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:10.025940 4553 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1565 milliseconds
11:54:10.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:10.025952 4543 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
11:54:10.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:10.027300 4543 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:54:10.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:10.029964 4543 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:54:10.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:10.031366 4543 hybrid_clock.cc:648] HybridClock initialized: now 1790682850031338 us; error 42 us; skew 500 ppm
11:54:10.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:10.032224 4543 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:54:10.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:54:10 dist-test-slave-cvvz krb5kdc[4097](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682850, etypes {rep=17 tkt=17 ses=17}, kudu/127.3.253.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:54:10.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:10.038419 4543 init.cc:377] Logged in from keytab as kudu/127.3.253.61@KRBTEST.COM (short username kudu)
11:54:10.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:10.043960 4543 webserver.cc:460] Webserver started at http://127.3.253.61:40643/ using document root <none> and password file <none>
11:54:10.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:10.045033 4543 fs_manager.cc:359] Metadata directory not provided
11:54:10.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:10.045390 4543 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:54:10.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:10.053956 4543 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.005s sys 0.002s
11:54:10.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:10.058843 4568 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:54:10.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:10.060036 4543 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.001s
11:54:10.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:10.060452 4543 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-1/data,/tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-1/wal
11:54:10.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1f532b79ad82484685a99d7995bf7070"
11:54:10.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:54:00 on dist-test-slave-cvvz"
11:54:10.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:10.062862 4543 fs_report.cc:383] FS layout report
11:54:10.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:54:10.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-1/wal
11:54:10.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-1/wal
11:54:10.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-1/data/data
11:54:10.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:54:10.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:54:10.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:54:10.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:54:10.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:54:10.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:54:10.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:54:10.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:54:10.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:54:10.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:54:10.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:54:10.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:10.106351 4543 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:54:10.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:10.112422 4543 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:54:10.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:10.112982 4543 kserver.cc:163] Server-wide thread pool size limit: 3276
11:54:10.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:10.164350 4462 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:54:10.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:10.165330 4462 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:54:10.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:10.198494 4543 rpc_server.cc:307] RPC server started. Bound to: 127.3.253.61:46581
11:54:10.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:10.198768 4620 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.3.253.61:46581 every 8 connection(s)
11:54:10.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:10.201973 4543 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-1/data/info.pb
11:54:10.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:10.205776 4084 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskrtJWFm/build/tsan/bin/kudu as pid 4543
11:54:10.206 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:54:10.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:10.206802 4377 master_runner.cc:186] Error getting master registration for 127.3.253.62:46169: Network error: Client connection negotiation failed: client connection to 127.3.253.62:46169: connect: Connection refused (error 111),
11:54:10.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:10.207214 4377 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
11:54:10.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:10.215147 4622 sys_catalog.cc:263] Verifying existing consensus state
11:54:10.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260929 11:54:10.215986 4622 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
11:54:10.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:10.216461 4543 master.cc:537] Master@127.3.253.61:46581 shutting down...
11:54:10.266 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.3.253.60:33837(127.3.253.60:33837)] connecting to peer
11:54:10.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:10.291436 4543 master.cc:552] Master@127.3.253.61:46581 shutdown complete.
11:54:10.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskrtJWFm/test-tmp/mini-kudu-cluster1269611041051101713/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
11:54:10.336 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.3.253.62:46169(127.3.253.62:46169)] connecting to peer
11:54:10.338 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.3.253.61:46581(127.3.253.61:46581)] connecting to peer
11:54:10.403 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:10.403 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.3.253.60:33837(127.3.253.60:33837)] Successfully connected to peer
11:54:10.404 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.253.62:46169(127.3.253.62:46169)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:10.408 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.253.62:46169: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:10.469 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:10.469 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.253.61:46581(127.3.253.61:46581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:10.470 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.253.61:46581: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:10.488 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
11:54:10.498 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.3.253.60
11:54:10.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:54:10 dist-test-slave-cvvz krb5kdc[4097](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682842, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.3.253.60@KRBTEST.COM
11:54:10.745 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xa47e8986, L:/127.0.0.1:46272 - R:/127.3.253.60:33837] using SASL/GSSAPI
11:54:10.760 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581; will retry
11:54:10.774 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.253.62:46169(127.3.253.62:46169)] connecting to peer
11:54:10.777 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.253.61:46581(127.3.253.61:46581)] connecting to peer
11:54:10.777 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:10.777 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.253.62:46169(127.3.253.62:46169)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:10.778 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.253.62:46169: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:10.781 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:10.781 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.253.61:46581(127.3.253.61:46581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:10.782 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.253.61:46581: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:10.783 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581; will retry
11:54:10.793 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.253.62:46169(127.3.253.62:46169)] connecting to peer
11:54:10.794 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.253.61:46581(127.3.253.61:46581)] connecting to peer
11:54:10.797 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:10.797 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.253.62:46169(127.3.253.62:46169)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:10.798 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.253.62:46169: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:10.800 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:10.800 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.253.61:46581(127.3.253.61:46581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:10.801 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.253.61:46581: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:10.802 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581; will retry
11:54:10.813 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.253.62:46169(127.3.253.62:46169)] connecting to peer
11:54:10.815 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.253.61:46581(127.3.253.61:46581)] connecting to peer
11:54:10.816 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:10.816 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.253.62:46169(127.3.253.62:46169)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:10.817 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.253.62:46169: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:10.819 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:10.819 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.253.61:46581(127.3.253.61:46581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:10.820 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.253.61:46581: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:10.820 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581; will retry
11:54:10.853 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.253.62:46169(127.3.253.62:46169)] connecting to peer
11:54:10.855 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.253.61:46581(127.3.253.61:46581)] connecting to peer
11:54:10.856 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:10.856 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.253.62:46169(127.3.253.62:46169)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:10.857 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.253.62:46169: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:10.860 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:10.861 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.253.61:46581(127.3.253.61:46581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:10.861 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.253.61:46581: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:10.862 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581; will retry
11:54:10.893 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.253.62:46169(127.3.253.62:46169)] connecting to peer
11:54:10.894 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.253.61:46581(127.3.253.61:46581)] connecting to peer
11:54:10.896 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:10.896 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.253.62:46169(127.3.253.62:46169)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:10.896 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.253.62:46169: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:10.899 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:10.899 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.253.61:46581(127.3.253.61:46581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:10.900 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.253.61:46581: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:10.900 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581; will retry
11:54:10.953 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.253.62:46169(127.3.253.62:46169)] connecting to peer
11:54:10.955 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.253.61:46581(127.3.253.61:46581)] connecting to peer
11:54:10.957 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:10.958 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.253.62:46169(127.3.253.62:46169)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:10.958 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.253.62:46169: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:10.959 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:10.959 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.253.61:46581(127.3.253.61:46581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:10.960 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.253.61:46581: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:10.960 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581; will retry
11:54:11.093 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.253.62:46169(127.3.253.62:46169)] connecting to peer
11:54:11.094 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.253.61:46581(127.3.253.61:46581)] connecting to peer
11:54:11.095 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:11.095 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.253.62:46169(127.3.253.62:46169)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:11.098 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.253.62:46169: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:11.099 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:11.099 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.253.61:46581(127.3.253.61:46581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:11.100 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.253.61:46581: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:11.101 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581; will retry
11:54:11.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:11.168859 4462 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:54:11.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:11.169191 4462 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:54:11.333 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.253.62:46169(127.3.253.62:46169)] connecting to peer
11:54:11.334 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.253.61:46581(127.3.253.61:46581)] connecting to peer
11:54:11.337 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:11.337 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.253.62:46169(127.3.253.62:46169)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:11.338 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.253.62:46169: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:11.340 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:11.340 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.253.61:46581(127.3.253.61:46581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:11.341 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.253.61:46581: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:11.341 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581; will retry
11:54:11.793 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.253.62:46169(127.3.253.62:46169)] connecting to peer
11:54:11.795 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.253.61:46581(127.3.253.61:46581)] connecting to peer
11:54:11.796 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:11.796 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.253.62:46169(127.3.253.62:46169)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:11.797 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.253.62:46169: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:11.797 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:11.798 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.253.61:46581(127.3.253.61:46581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:11.798 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.253.61:46581: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:11.800 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581; will retry
11:54:11.892 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.253.62:46169(127.3.253.62:46169)] connecting to peer
11:54:11.894 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.253.61:46581(127.3.253.61:46581)] connecting to peer
11:54:11.895 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:11.895 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.253.62:46169(127.3.253.62:46169)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:11.895 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.253.62:46169: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:11.897 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:11.897 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.253.61:46581(127.3.253.61:46581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:11.898 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.253.61:46581: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:11.898 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581; will retry
11:54:12.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:12.172257 4462 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:54:12.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:12.172493 4462 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:54:12.432 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.253.62:46169(127.3.253.62:46169)] connecting to peer
11:54:12.434 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.253.61:46581(127.3.253.61:46581)] connecting to peer
11:54:12.435 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:12.435 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.253.62:46169(127.3.253.62:46169)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:12.435 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.253.62:46169: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:12.437 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:12.437 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.253.61:46581(127.3.253.61:46581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:12.437 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.253.61:46581: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:12.437 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581; will retry
11:54:13.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:13.175362 4462 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:54:13.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:13.175633 4462 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:54:13.632 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.253.62:46169(127.3.253.62:46169)] connecting to peer
11:54:13.634 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.253.61:46581(127.3.253.61:46581)] connecting to peer
11:54:13.635 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:13.636 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.253.62:46169(127.3.253.62:46169)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:13.636 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.253.62:46169: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:13.637 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:13.637 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.253.61:46581(127.3.253.61:46581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:13.638 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.253.61:46581: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:13.639 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581; will retry
11:54:13.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:13.917994 4631 raft_consensus.cc:482] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
11:54:13.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:13.918673 4631 raft_consensus.cc:504] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8ed01ddce3475db4624669642fe48e" member_type: VOTER last_known_addr { host: "127.3.253.62" port: 46169 } } peers { permanent_uuid: "1f532b79ad82484685a99d7995bf7070" member_type: VOTER last_known_addr { host: "127.3.253.61" port: 46581 } } peers { permanent_uuid: "28f41463e82d4a36b1ff698a18ec239d" member_type: VOTER last_known_addr { host: "127.3.253.60" port: 33837 } }
11:54:13.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:13.920799 4631 leader_election.cc:288] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 9e8ed01ddce3475db4624669642fe48e (127.3.253.62:46169), 1f532b79ad82484685a99d7995bf7070 (127.3.253.61:46581)
11:54:13.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:13.923830 4395 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.253.62:46169: connect: Connection refused (error 111) [suppressed 5 similar messages]
11:54:13.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:13.926800 4395 leader_election.cc:334] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 9e8ed01ddce3475db4624669642fe48e (127.3.253.62:46169): Network error: Client connection negotiation failed: client connection to 127.3.253.62:46169: connect: Connection refused (error 111)
11:54:13.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:13.927459 4395 leader_election.cc:334] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 1f532b79ad82484685a99d7995bf7070 (127.3.253.61:46581): Network error: Client connection negotiation failed: client connection to 127.3.253.61:46581: connect: Connection refused (error 111)
11:54:13.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:13.927758 4395 leader_election.cc:302] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 28f41463e82d4a36b1ff698a18ec239d; no voters: 1f532b79ad82484685a99d7995bf7070, 9e8ed01ddce3475db4624669642fe48e
11:54:13.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:13.928464 4631 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
11:54:14.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:14.178900 4462 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:54:14.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:14.179173 4462 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:54:14.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:14.273257 3586 debug-util.cc:398] Leaking SignalData structure 0x7b08000ae600 after lost signal to thread 3517
11:54:14.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:14.273810 3586 debug-util.cc:398] Leaking SignalData structure 0x7b08000c1840 after lost signal to thread 3589
11:54:14.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:54:14 dist-test-slave-cvvz krb5kdc[4097](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682854, etypes {rep=17 tkt=17 ses=17}, kudu/127.3.253.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:54:14.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:14.753012 4390 init.cc:295] Successfully reacquired a new kerberos TGT
11:54:15.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:15.182296 4462 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:54:15.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:15.182579 4462 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:54:15.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:15.211140 4377 master_runner.cc:186] Error getting master registration for 127.3.253.62:46169: Network error: Client connection negotiation failed: client connection to 127.3.253.62:46169: connect: Connection refused (error 111),
11:54:15.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:15.211694 4377 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
11:54:16.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:16.185469 4462 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:54:16.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:16.185753 4462 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:54:16.292 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.253.62:46169(127.3.253.62:46169)] connecting to peer
11:54:16.293 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.253.61:46581(127.3.253.61:46581)] connecting to peer
11:54:16.294 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:16.294 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.253.62:46169(127.3.253.62:46169)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:16.295 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.253.62:46169: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:16.295 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:16.295 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.253.61:46581(127.3.253.61:46581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:16.295 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.253.61:46581: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:16.296 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581; will retry
11:54:17.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:17.188741 4462 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:54:17.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:17.189069 4462 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:54:17.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:17.773000 4638 raft_consensus.cc:482] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
11:54:17.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:17.773495 4638 raft_consensus.cc:504] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8ed01ddce3475db4624669642fe48e" member_type: VOTER last_known_addr { host: "127.3.253.62" port: 46169 } } peers { permanent_uuid: "1f532b79ad82484685a99d7995bf7070" member_type: VOTER last_known_addr { host: "127.3.253.61" port: 46581 } } peers { permanent_uuid: "28f41463e82d4a36b1ff698a18ec239d" member_type: VOTER last_known_addr { host: "127.3.253.60" port: 33837 } }
11:54:17.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:17.775108 4638 leader_election.cc:288] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 9e8ed01ddce3475db4624669642fe48e (127.3.253.62:46169), 1f532b79ad82484685a99d7995bf7070 (127.3.253.61:46581)
11:54:17.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:17.780503 4395 leader_election.cc:334] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 9e8ed01ddce3475db4624669642fe48e (127.3.253.62:46169): Network error: Client connection negotiation failed: client connection to 127.3.253.62:46169: connect: Connection refused (error 111)
11:54:17.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:17.781153 4395 leader_election.cc:334] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 1f532b79ad82484685a99d7995bf7070 (127.3.253.61:46581): Network error: Client connection negotiation failed: client connection to 127.3.253.61:46581: connect: Connection refused (error 111)
11:54:17.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:17.781497 4395 leader_election.cc:302] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 28f41463e82d4a36b1ff698a18ec239d; no voters: 1f532b79ad82484685a99d7995bf7070, 9e8ed01ddce3475db4624669642fe48e
11:54:17.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:17.782014 4638 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
11:54:18.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:18.192603 4462 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:54:18.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:18.192916 4462 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:54:19.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:19.195859 4462 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:54:19.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:19.196159 4462 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:54:19.633 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.253.62:46169(127.3.253.62:46169)] connecting to peer
11:54:19.635 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.253.61:46581(127.3.253.61:46581)] connecting to peer
11:54:19.636 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:19.636 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.253.62:46169(127.3.253.62:46169)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:19.637 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.253.62:46169: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:19.639 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:19.639 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.253.61:46581(127.3.253.61:46581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:19.639 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.253.61:46581: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:19.639 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581; will retry
11:54:20.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:20.199728 4462 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:54:20.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:20.199982 4462 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:54:20.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:20.215116 4377 master_runner.cc:186] Error getting master registration for 127.3.253.62:46169: Network error: Client connection negotiation failed: client connection to 127.3.253.62:46169: connect: Connection refused (error 111),
11:54:20.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:20.215556 4377 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
11:54:20.292 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.253.62:46169(127.3.253.62:46169)] connecting to peer
11:54:20.295 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.253.61:46581(127.3.253.61:46581)] connecting to peer
11:54:20.295 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:20.295 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.253.62:46169(127.3.253.62:46169)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:20.295 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.253.62:46169: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:20.298 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:20.299 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.253.61:46581(127.3.253.61:46581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:20.299 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.253.61:46581: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:20.299 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581; will retry
11:54:21.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:21.203229 4462 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:54:21.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:21.203464 4462 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:54:21.353 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.253.62:46169(127.3.253.62:46169)] connecting to peer
11:54:21.355 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.253.61:46581(127.3.253.61:46581)] connecting to peer
11:54:21.355 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:21.355 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.253.62:46169(127.3.253.62:46169)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:21.356 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.253.62:46169: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:21.356 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:21.356 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.253.61:46581(127.3.253.61:46581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:21.356 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.253.61:46581: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:21.357 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581; will retry
11:54:22.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:22.206564 4462 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:54:22.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:22.206830 4462 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:54:22.893 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.253.62:46169(127.3.253.62:46169)] connecting to peer
11:54:22.894 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.253.61:46581(127.3.253.61:46581)] connecting to peer
11:54:22.895 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:22.895 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.253.62:46169(127.3.253.62:46169)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:22.895 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.253.62:46169: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:22.898 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:22.898 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.253.61:46581(127.3.253.61:46581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:22.898 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.253.61:46581: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:22.898 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581; will retry
11:54:23.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:23.184418 4649 raft_consensus.cc:482] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
11:54:23.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:23.184868 4649 raft_consensus.cc:504] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8ed01ddce3475db4624669642fe48e" member_type: VOTER last_known_addr { host: "127.3.253.62" port: 46169 } } peers { permanent_uuid: "1f532b79ad82484685a99d7995bf7070" member_type: VOTER last_known_addr { host: "127.3.253.61" port: 46581 } } peers { permanent_uuid: "28f41463e82d4a36b1ff698a18ec239d" member_type: VOTER last_known_addr { host: "127.3.253.60" port: 33837 } }
11:54:23.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:23.186509 4649 leader_election.cc:288] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 9e8ed01ddce3475db4624669642fe48e (127.3.253.62:46169), 1f532b79ad82484685a99d7995bf7070 (127.3.253.61:46581)
11:54:23.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:23.189726 4395 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.253.62:46169: connect: Connection refused (error 111) [suppressed 3 similar messages]
11:54:23.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:23.192866 4395 leader_election.cc:334] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 9e8ed01ddce3475db4624669642fe48e (127.3.253.62:46169): Network error: Client connection negotiation failed: client connection to 127.3.253.62:46169: connect: Connection refused (error 111)
11:54:23.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:23.193492 4395 leader_election.cc:334] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 1f532b79ad82484685a99d7995bf7070 (127.3.253.61:46581): Network error: Client connection negotiation failed: client connection to 127.3.253.61:46581: connect: Connection refused (error 111)
11:54:23.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:23.193760 4395 leader_election.cc:302] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 28f41463e82d4a36b1ff698a18ec239d; no voters: 1f532b79ad82484685a99d7995bf7070, 9e8ed01ddce3475db4624669642fe48e
11:54:23.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:23.194341 4649 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
11:54:23.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:23.209872 4462 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:54:23.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:23.210187 4462 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:54:24.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:24.214116 4462 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:54:24.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:24.214480 4462 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:54:24.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:54:24 dist-test-slave-cvvz krb5kdc[4097](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682864, etypes {rep=17 tkt=17 ses=17}, kudu/127.3.253.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:54:24.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:24.755722 4390 init.cc:295] Successfully reacquired a new kerberos TGT
11:54:25.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:25.218245 4462 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:54:25.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:25.218654 4462 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:54:25.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:25.220324 4377 master_runner.cc:186] Error getting master registration for 127.3.253.62:46169: Network error: Client connection negotiation failed: client connection to 127.3.253.62:46169: connect: Connection refused (error 111),
11:54:25.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:25.220777 4377 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
11:54:25.313 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.253.62:46169(127.3.253.62:46169)] connecting to peer
11:54:25.315 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.253.61:46581(127.3.253.61:46581)] connecting to peer
11:54:25.315 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:25.315 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.253.62:46169(127.3.253.62:46169)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:25.316 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:25.316 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.253.62:46169: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:25.316 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.253.61:46581(127.3.253.61:46581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:25.316 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.253.61:46581: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:25.318 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581; will retry
11:54:26.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:26.222065 4462 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:54:26.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:26.222389 4462 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:54:26.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:26.584824 4656 raft_consensus.cc:482] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
11:54:26.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:26.585361 4656 raft_consensus.cc:504] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e8ed01ddce3475db4624669642fe48e" member_type: VOTER last_known_addr { host: "127.3.253.62" port: 46169 } } peers { permanent_uuid: "1f532b79ad82484685a99d7995bf7070" member_type: VOTER last_known_addr { host: "127.3.253.61" port: 46581 } } peers { permanent_uuid: "28f41463e82d4a36b1ff698a18ec239d" member_type: VOTER last_known_addr { host: "127.3.253.60" port: 33837 } }
11:54:26.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:26.587229 4656 leader_election.cc:288] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 9e8ed01ddce3475db4624669642fe48e (127.3.253.62:46169), 1f532b79ad82484685a99d7995bf7070 (127.3.253.61:46581)
11:54:26.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:26.592775 4395 leader_election.cc:334] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 9e8ed01ddce3475db4624669642fe48e (127.3.253.62:46169): Network error: Client connection negotiation failed: client connection to 127.3.253.62:46169: connect: Connection refused (error 111)
11:54:26.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:26.593649 4395 leader_election.cc:334] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 1f532b79ad82484685a99d7995bf7070 (127.3.253.61:46581): Network error: Client connection negotiation failed: client connection to 127.3.253.61:46581: connect: Connection refused (error 111)
11:54:26.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:26.594017 4395 leader_election.cc:302] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 28f41463e82d4a36b1ff698a18ec239d; no voters: 1f532b79ad82484685a99d7995bf7070, 9e8ed01ddce3475db4624669642fe48e
11:54:26.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:26.594755 4656 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
11:54:27.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:27.225292 4462 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:54:27.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:27.225587 4462 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:54:28.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:28.228727 4462 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:54:28.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:28.229044 4462 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:54:29.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:29.232105 4462 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:54:29.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:29.232434 4462 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:54:29.253 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.253.62:46169(127.3.253.62:46169)] connecting to peer
11:54:29.254 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.253.61:46581(127.3.253.61:46581)] connecting to peer
11:54:29.257 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:29.258 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.253.62:46169(127.3.253.62:46169)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:29.258 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.253.62:46169: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:29.258 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:29.258 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.253.61:46581(127.3.253.61:46581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:29.259 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.253.61:46581: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:29.259 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581; will retry
11:54:30.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:30.223873 4377 master_runner.cc:186] Error getting master registration for 127.3.253.62:46169: Network error: Client connection negotiation failed: client connection to 127.3.253.62:46169: connect: Connection refused (error 111),
11:54:30.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:30.224314 4377 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
11:54:30.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:30.236361 4462 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:54:30.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:30.236773 4462 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:54:31.073 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.253.62:46169(127.3.253.62:46169)] connecting to peer
11:54:31.074 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.253.61:46581(127.3.253.61:46581)] connecting to peer
11:54:31.074 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:31.074 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.253.62:46169(127.3.253.62:46169)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:31.075 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.253.62:46169: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:31.076 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:31.076 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.253.61:46581(127.3.253.61:46581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:31.076 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.253.61:46581: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:31.077 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581; will retry
11:54:31.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:31.239540 4462 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:54:31.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:31.239856 4462 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:54:31.313 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.253.62:46169(127.3.253.62:46169)] connecting to peer
11:54:31.314 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.253.61:46581(127.3.253.61:46581)] connecting to peer
11:54:31.314 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:31.314 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.253.62:46169(127.3.253.62:46169)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:31.314 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.253.62:46169: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:31.315 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:31.315 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.253.61:46581(127.3.253.61:46581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:31.315 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.253.61:46581: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:31.316 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581; will retry
11:54:32.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:32.242931 4462 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:54:32.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:32.243259 4462 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:54:32.672 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.253.62:46169(127.3.253.62:46169)] connecting to peer
11:54:32.673 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.253.61:46581(127.3.253.61:46581)] connecting to peer
11:54:32.674 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:32.674 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.253.62:46169(127.3.253.62:46169)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:32.674 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.253.62:46169: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:32.674 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:32.674 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.253.61:46581(127.3.253.61:46581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:32.675 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.253.61:46581: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:32.676 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581; will retry
11:54:33.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:33.246137 4462 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:54:33.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:33.246421 4462 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:54:34.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:34.250285 4462 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:54:34.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:34.250655 4462 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:54:34.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:54:34 dist-test-slave-cvvz krb5kdc[4097](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682874, etypes {rep=17 tkt=17 ses=17}, kudu/127.3.253.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:54:34.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:34.758652 4390 init.cc:295] Successfully reacquired a new kerberos TGT
11:54:35.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:35.228211 4377 master_runner.cc:186] Error getting master registration for 127.3.253.62:46169: Network error: Client connection negotiation failed: client connection to 127.3.253.62:46169: connect: Connection refused (error 111),
11:54:35.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:35.228629 4377 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
11:54:35.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:35.254042 4462 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:54:35.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:35.254451 4462 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:54:36.173 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.253.62:46169(127.3.253.62:46169)] connecting to peer
11:54:36.173 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.253.61:46581(127.3.253.61:46581)] connecting to peer
11:54:36.174 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:36.174 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.253.61:46581(127.3.253.61:46581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:36.175 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.253.61:46581: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:36.175 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:36.175 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.253.62:46169(127.3.253.62:46169)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:36.175 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.253.62:46169: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:36.177 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581; will retry
11:54:36.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:36.257192 4462 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 28f41463e82d4a36b1ff698a18ec239d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:54:36.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:54:36.257480 4462 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:54:36.973 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.253.62:46169(127.3.253.62:46169)] connecting to peer
11:54:36.974 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.253.61:46581(127.3.253.61:46581)] connecting to peer
11:54:36.974 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:36.975 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.253.62:46169(127.3.253.62:46169)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:36.975 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.253.62:46169: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:36.976 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:36.976 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.253.61:46581(127.3.253.61:46581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:36.976 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.253.61:46581: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:36.977 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581; will retry
11:54:37.113 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.253.62:46169(127.3.253.62:46169)] connecting to peer
11:54:37.114 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.253.61:46581(127.3.253.61:46581)] connecting to peer
11:54:37.115 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:37.115 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:37.115 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.253.61:46581(127.3.253.61:46581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:37.115 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.253.62:46169(127.3.253.62:46169)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:37.115 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.253.61:46581: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581
11:54:37.115 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.253.62:46169: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
11:54:37.116 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581; will retry
11:54:37.121 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=26906), Traces: [0ms] refreshing cache from master, [36ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.60:33837, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.62:46169, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.61:46581, [190ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.62:46169: Network error: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169, [254ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.61:46581: Network error: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [541ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.60:33837: OK, [545ms] delaying RPC due to: Service unavailable: Master config (127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [556ms] refreshing cache from master, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.60:33837, [558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.62:46169, [560ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.61:46581, [562ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.62:46169: Network error: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169, [564ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.60:33837: OK, [566ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.61:46581: Network error: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [567ms] delaying RPC due to: Service unavailable: Master config (127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [576ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.60:33837, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.62:46169, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.61:46581, [582ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.62:46169: Network error: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169, [583ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.60:33837: OK, [585ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.61:46581: Network error: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [586ms] delaying RPC due to: Service unavailable: Master config (127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [596ms] refreshing cache from master, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.60:33837, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.62:46169, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.61:46581, [601ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.62:46169: Network error: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169, [602ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.60:33837: OK, [603ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.61:46581: Network error: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [605ms] delaying RPC due to: Service unavailable: Master config (127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [636ms] refreshing cache from master, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.60:33837, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.62:46169, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.61:46581, [640ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.62:46169: Network error: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169, [644ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.60:33837: OK, [645ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.61:46581: Network error: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [646ms] delaying RPC due to: Service unavailable: Master config (127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [676ms] refreshing cache from master, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.60:33837, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.62:46169, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.61:46581, [680ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.62:46169: Network error: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169, [683ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.60:33837: OK, [683ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.61:46581: Network error: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [685ms] delaying RPC due to: Service unavailable: Master config (127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [736ms] refreshing cache from master, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.60:33837, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.62:46169, [739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.61:46581, [742ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.62:46169: Network error: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169, [742ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.60:33837: OK, [743ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.61:46581: Network error: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [744ms] delaying RPC due to: Service unavailable: Master config (127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [876ms] refreshing cache from master, [876ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.60:33837, [876ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.62:46169, [878ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.61:46581, [880ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.62:46169: Network error: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169, [880ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.60:33837: OK, [884ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.61:46581: Network error: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [885ms] delaying RPC due to: Service unavailable: Master config (127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [1116ms] refreshing cache from master, [1116ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.60:33837, [1117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.62:46169, [1118ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.61:46581, [1122ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.62:46169: Network error: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169, [1123ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.60:33837: OK, [1124ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.61:46581: Network error: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [1125ms] delaying RPC due to: Service unavailable: Master config (127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [1576ms] refreshing cache from master, [1576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.60:33837, [1577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.62:46169, [1578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.61:46581, [1580ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.62:46169: Network error: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169, [1582ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.61:46581: Network error: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [1583ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.60:33837: OK, [1584ms] delaying RPC due to: Service unavailable: Master config (127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [1676ms] refreshing cache from master, [1676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.60:33837, [1676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.62:46169, [1677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.61:46581, [1679ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.62:46169: Network error: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169, [1680ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.60:33837: OK, [1681ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.61:46581: Network error: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [1682ms] delaying RPC due to: Service unavailable: Master config (127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [2216ms] refreshing cache from master, [2216ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.60:33837, [2216ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.62:46169, [2217ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.61:46581, [2219ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.62:46169: Network error: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169, [2221ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.60:33837: OK, [2221ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.61:46581: Network error: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [2221ms] delaying RPC due to: Service unavailable: Master config (127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [3416ms] refreshing cache from master, [3416ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.60:33837, [3416ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.62:46169, [3418ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.61:46581, [3420ms] too many traces: truncated at 100 traces, deferred=Deferred@760482748(state=PENDING, result=null, callback=(continuation of Deferred@1607012979 after retry RPC after error@100652726) -> (continuation of Deferred@370998217 after retry RPC after error@462953309) -> (continuation of Deferred@206952663 after retry RPC after error@2003105165) -> (continuation of Deferred@106400283 after retry RPC after error@1565404031) -> (continuation of Deferred@2139079521 after retry RPC after error@803707294) -> (continuation of Deferred@366659883 after retry RPC after error@962792764) -> (continuation of Deferred@948352442 after retry RPC after error@733933661) -> (continuation of Deferred@1760491408 after retry RPC after error@1229262172) -> (continuation of Deferred@1147357867 after retry RPC after error@411264610) -> (continuation of Deferred@122572949 after retry RPC after error@1711313613) -> (continuation of Deferred@946387636 after retry RPC after error@995851181) -> (continuation of Deferred@1323589291 after retry RPC after error@739257152) -> (continuation of Deferred@1372625260 after retry RPC after error@825628392) -> (continuation of Deferred@1573590883 after retry RPC after error@1890352561) -> (continuation of Deferred@635605297 after retry RPC after error@610419251) -> (continuation of Deferred@555170259 after retry RPC after error@843418037) -> (continuation of Deferred@52428245 after retry RPC after error@895112287) -> (continuation of Deferred@756755078 after retry RPC after error@856495793) -> (continuation of Deferred@1325442128 after retry RPC after error@673184067) -> (continuation of Deferred@1545910748 after retry RPC after error@122127479) -> (continuation of Deferred@1294959339 after retry RPC after error@447594476) -> (continuation of Deferred@1673400124 after retry RPC after error@1491159808) -> (continuation of Deferred@1776857645 after retry RPC after error@516760099) -> (continuation of Deferred@219748649 after retry RPC after error@1601370817) -> (continuation of Deferred@1940345560 after retry RPC after error@76327605), errback=(continuation of Deferred@1607012979 after retry RPC after error@100652726) -> (continuation of Deferred@370998217 after retry RPC after error@462953309) -> (continuation of Deferred@206952663 after retry RPC after error@2003105165) -> (continuation of Deferred@106400283 after retry RPC after error@1565404031) -> (continuation of Deferred@2139079521 after retry RPC after error@803707294) -> (continuation of Deferred@366659883 after retry RPC after error@962792764) -> (continuation of Deferred@948352442 after retry RPC after error@733933661) -> (continuation of Deferred@1760491408 after retry RPC after error@1229262172) -> (continuation of Deferred@1147357867 after retry RPC after error@411264610) -> (continuation of Deferred@122572949 after retry RPC after error@1711313613) -> (continuation of Deferred@946387636 after retry RPC after error@995851181) -> (continuation of Deferred@1323589291 after retry RPC after error@739257152) -> (continuation of Deferred@1372625260 after retry RPC after error@825628392) -> (continuation of Deferred@1573590883 after retry RPC after error@1890352561) -> (continuation of Deferred@635605297 after retry RPC after error@610419251) -> (continuation of Deferred@555170259 after retry RPC after error@843418037) -> (continuation of Deferred@52428245 after retry RPC after error@895112287) -> (continuation of Deferred@756755078 after retry RPC after error@856495793) -> (continuation of Deferred@1325442128 after retry RPC after error@673184067) -> (continuation of Deferred@1545910748 after retry RPC after error@122127479) -> (continuation of Deferred@1294959339 after retry RPC after error@447594476) -> (continuation of Deferred@1673400124 after retry RPC after error@1491159808) -> (continuation of Deferred@1776857645 after retry RPC after error@516760099) -> (continuation of Deferred@219748649 after retry RPC after error@1601370817) -> (continuation of Deferred@1940345560 after retry RPC after error@76327605)))
11:54:37.126 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.3.253.62:46169(127.3.253.62:46169)] disconnecting while in state TERMINATED
11:54:37.127 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.3.253.61:46581(127.3.253.61:46581)] disconnecting while in state TERMINATED
11:54:37.127 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.3.253.60:33837(127.3.253.60:33837)] disconnecting while in state READY
11:54:37.133 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
11:54:37.135 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.3.253.60:33837(127.3.253.60:33837)] handling channelInactive
11:54:37.135 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.253.60:33837(127.3.253.60:33837)] cleaning up while in state READY due to: connection closed
11:54:37.142 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
11:54:37.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:37.146628 4084 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskrtJWFm/build/tsan/bin/kudu with pid 4377
11:54:38.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T11:54:38Z chronyd exiting
11:54:39.418 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
11:54:39.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:39.420320 3355 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskrtJWFm/build/tsan/bin/kudu with pid 3637
11:54:39.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:39.446830 3355 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskrtJWFm/build/tsan/bin/kudu with pid 3802
11:54:39.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:39.474830 3355 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskrtJWFm/build/tsan/bin/kudu with pid 3942
11:54:39.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:39.514599 3355 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskrtJWFm/build/tsan/bin/kudu with pid 3367
11:54:39.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:39.548362 3355 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskrtJWFm/build/tsan/bin/kudu with pid 3437
11:54:39.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:54:39.581561 3355 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskrtJWFm/build/tsan/bin/kudu with pid 3512
11:54:40.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T11:54:40Z chronyd exiting
11:54:41.792 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=26906), Traces: [0ms] refreshing cache from master, [36ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.60:33837, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.62:46169, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.61:46581, [190ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.62:46169: Network error: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169, [254ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.61:46581: Network error: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [541ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.60:33837: OK, [545ms] delaying RPC due to: Service unavailable: Master config (127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [556ms] refreshing cache from master, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.60:33837, [558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.62:46169, [560ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.61:46581, [562ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.62:46169: Network error: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169, [564ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.60:33837: OK, [566ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.61:46581: Network error: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [567ms] delaying RPC due to: Service unavailable: Master config (127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [576ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.60:33837, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.62:46169, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.61:46581, [582ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.62:46169: Network error: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169, [583ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.60:33837: OK, [585ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.61:46581: Network error: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [586ms] delaying RPC due to: Service unavailable: Master config (127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [596ms] refreshing cache from master, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.60:33837, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.62:46169, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.61:46581, [601ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.62:46169: Network error: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169, [602ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.60:33837: OK, [603ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.61:46581: Network error: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [605ms] delaying RPC due to: Service unavailable: Master config (127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [636ms] refreshing cache from master, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.60:33837, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.62:46169, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.61:46581, [640ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.62:46169: Network error: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169, [644ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.60:33837: OK, [645ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.61:46581: Network error: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [646ms] delaying RPC due to: Service unavailable: Master config (127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [676ms] refreshing cache from master, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.60:33837, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.62:46169, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.61:46581, [680ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.62:46169: Network error: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169, [683ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.60:33837: OK, [683ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.61:46581: Network error: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [685ms] delaying RPC due to: Service unavailable: Master config (127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [736ms] refreshing cache from master, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.60:33837, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.62:46169, [739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.61:46581, [742ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.62:46169: Network error: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169, [742ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.60:33837: OK, [743ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.61:46581: Network error: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [744ms] delaying RPC due to: Service unavailable: Master config (127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [876ms] refreshing cache from master, [876ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.60:33837, [876ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.62:46169, [878ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.61:46581, [880ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.62:46169: Network error: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169, [880ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.60:33837: OK, [884ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.61:46581: Network error: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [885ms] delaying RPC due to: Service unavailable: Master config (127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [1116ms] refreshing cache from master, [1116ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.60:33837, [1117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.62:46169, [1118ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.61:46581, [1122ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.62:46169: Network error: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169, [1123ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.60:33837: OK, [1124ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.61:46581: Network error: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [1125ms] delaying RPC due to: Service unavailable: Master config (127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [1576ms] refreshing cache from master, [1576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.60:33837, [1577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.62:46169, [1578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.61:46581, [1580ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.62:46169: Network error: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169, [1582ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.61:46581: Network error: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [1583ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.60:33837: OK, [1584ms] delaying RPC due to: Service unavailable: Master config (127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [1676ms] refreshing cache from master, [1676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.60:33837, [1676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.62:46169, [1677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.61:46581, [1679ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.62:46169: Network error: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169, [1680ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.60:33837: OK, [1681ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.61:46581: Network error: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [1682ms] delaying RPC due to: Service unavailable: Master config (127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [2216ms] refreshing cache from master, [2216ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.60:33837, [2216ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.62:46169, [2217ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.61:46581, [2219ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.62:46169: Network error: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169, [2221ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.60:33837: OK, [2221ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.61:46581: Network error: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [2221ms] delaying RPC due to: Service unavailable: Master config (127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [3416ms] refreshing cache from master, [3416ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.60:33837, [3416ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.62:46169, [3418ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.61:46581, [3420ms] too many traces: truncated at 100 traces, deferred=Deferred@760482748(state=PENDING, result=null, callback=(continuation of Deferred@1607012979 after retry RPC after error@100652726) -> (continuation of Deferred@370998217 after retry RPC after error@462953309) -> (continuation of Deferred@206952663 after retry RPC after error@2003105165) -> (continuation of Deferred@106400283 after retry RPC after error@1565404031) -> (continuation of Deferred@2139079521 after retry RPC after error@803707294) -> (continuation of Deferred@366659883 after retry RPC after error@962792764) -> (continuation of Deferred@948352442 after retry RPC after error@733933661) -> (continuation of Deferred@1760491408 after retry RPC after error@1229262172) -> (continuation of Deferred@1147357867 after retry RPC after error@411264610) -> (continuation of Deferred@122572949 after retry RPC after error@1711313613) -> (continuation of Deferred@946387636 after retry RPC after error@995851181) -> (continuation of Deferred@1323589291 after retry RPC after error@739257152) -> (continuation of Deferred@1372625260 after retry RPC after error@825628392) -> (continuation of Deferred@1573590883 after retry RPC after error@1890352561) -> (continuation of Deferred@635605297 after retry RPC after error@610419251) -> (continuation of Deferred@555170259 after retry RPC after error@843418037) -> (continuation of Deferred@52428245 after retry RPC after error@895112287) -> (continuation of Deferred@756755078 after retry RPC after error@856495793) -> (continuation of Deferred@1325442128 after retry RPC after error@673184067) -> (continuation of Deferred@1545910748 after retry RPC after error@122127479) -> (continuation of Deferred@1294959339 after retry RPC after error@447594476) -> (continuation of Deferred@1673400124 after retry RPC after error@1491159808) -> (continuation of Deferred@1776857645 after retry RPC after error@516760099) -> (continuation of Deferred@219748649 after retry RPC after error@1601370817) -> (continuation of Deferred@1940345560 after retry RPC after error@76327605), errback=(continuation of Deferred@1607012979 after retry RPC after error@100652726) -> (continuation of Deferred@370998217 after retry RPC after error@462953309) -> (continuation of Deferred@206952663 after retry RPC after error@2003105165) -> (continuation of Deferred@106400283 after retry RPC after error@1565404031) -> (continuation of Deferred@2139079521 after retry RPC after error@803707294) -> (continuation of Deferred@366659883 after retry RPC after error@962792764) -> (continuation of Deferred@948352442 after retry RPC after error@733933661) -> (continuation of Deferred@1760491408 after retry RPC after error@1229262172) -> (continuation of Deferred@1147357867 after retry RPC after error@411264610) -> (continuation of Deferred@122572949 after retry RPC after error@1711313613) -> (continuation of Deferred@946387636 after retry RPC after error@995851181) -> (continuation of Deferred@1323589291 after retry RPC after error@739257152) -> (continuation of Deferred@1372625260 after retry RPC after error@825628392) -> (continuation of Deferred@1573590883 after retry RPC after error@1890352561) -> (continuation of Deferred@635605297 after retry RPC after error@610419251) -> (continuation of Deferred@555170259 after retry RPC after error@843418037) -> (continuation of Deferred@52428245 after retry RPC after error@895112287) -> (continuation of Deferred@756755078 after retry RPC after error@856495793) -> (continuation of Deferred@1325442128 after retry RPC after error@673184067) -> (continuation of Deferred@1545910748 after retry RPC after error@122127479) -> (continuation of Deferred@1294959339 after retry RPC after error@447594476) -> (continuation of Deferred@1673400124 after retry RPC after error@1491159808) -> (continuation of Deferred@1776857645 after retry RPC after error@516760099) -> (continuation of Deferred@219748649 after retry RPC after error@1601370817) -> (continuation of Deferred@1940345560 after retry RPC after error@76327605)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.3.253.62:46169
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=26906), Traces: [0ms] refreshing cache from master, [36ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.60:33837, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.62:46169, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.61:46581, [190ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.62:46169: Network error: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169, [254ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.61:46581: Network error: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [541ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.60:33837: OK, [545ms] delaying RPC due to: Service unavailable: Master config (127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [556ms] refreshing cache from master, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.60:33837, [558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.62:46169, [560ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.61:46581, [562ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.62:46169: Network error: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169, [564ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.60:33837: OK, [566ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.61:46581: Network error: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [567ms] delaying RPC due to: Service unavailable: Master config (127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [576ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.60:33837, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.62:46169, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.61:46581, [582ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.62:46169: Network error: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169, [583ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.60:33837: OK, [585ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.61:46581: Network error: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [586ms] delaying RPC due to: Service unavailable: Master config (127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [596ms] refreshing cache from master, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.60:33837, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.62:46169, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.61:46581, [601ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.62:46169: Network error: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169, [602ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.60:33837: OK, [603ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.61:46581: Network error: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [605ms] delaying RPC due to: Service unavailable: Master config (127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [636ms] refreshing cache from master, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.60:33837, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.62:46169, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.61:46581, [640ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.62:46169: Network error: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169, [644ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.60:33837: OK, [645ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.61:46581: Network error: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [646ms] delaying RPC due to: Service unavailable: Master config (127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [676ms] refreshing cache from master, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.60:33837, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.62:46169, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.61:46581, [680ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.62:46169: Network error: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169, [683ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.60:33837: OK, [683ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.61:46581: Network error: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [685ms] delaying RPC due to: Service unavailable: Master config (127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [736ms] refreshing cache from master, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.60:33837, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.62:46169, [739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.61:46581, [742ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.62:46169: Network error: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169, [742ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.60:33837: OK, [743ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.61:46581: Network error: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [744ms] delaying RPC due to: Service unavailable: Master config (127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [876ms] refreshing cache from master, [876ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.60:33837, [876ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.62:46169, [878ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.61:46581, [880ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.62:46169: Network error: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169, [880ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.60:33837: OK, [884ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.61:46581: Network error: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [885ms] delaying RPC due to: Service unavailable: Master config (127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [1116ms] refreshing cache from master, [1116ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.60:33837, [1117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.62:46169, [1118ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.61:46581, [1122ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.62:46169: Network error: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169, [1123ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.60:33837: OK, [1124ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.61:46581: Network error: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [1125ms] delaying RPC due to: Service unavailable: Master config (127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [1576ms] refreshing cache from master, [1576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.60:33837, [1577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.62:46169, [1578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.61:46581, [1580ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.62:46169: Network error: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169, [1582ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.61:46581: Network error: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [1583ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.60:33837: OK, [1584ms] delaying RPC due to: Service unavailable: Master config (127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [1676ms] refreshing cache from master, [1676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.60:33837, [1676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.62:46169, [1677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.61:46581, [1679ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.62:46169: Network error: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169, [1680ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.60:33837: OK, [1681ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.61:46581: Network error: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [1682ms] delaying RPC due to: Service unavailable: Master config (127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [2216ms] refreshing cache from master, [2216ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.60:33837, [2216ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.62:46169, [2217ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.61:46581, [2219ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.62:46169: Network error: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169, [2221ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.60:33837: OK, [2221ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.61:46581: Network error: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [2221ms] delaying RPC due to: Service unavailable: Master config (127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [3416ms] refreshing cache from master, [3416ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.60:33837, [3416ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.62:46169, [3418ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.61:46581, [3420ms] too many traces: truncated at 100 traces, deferred=Deferred@760482748(state=PENDING, result=null, callback=(continuation of Deferred@1607012979 after retry RPC after error@100652726) -> (continuation of Deferred@370998217 after retry RPC after error@462953309) -> (continuation of Deferred@206952663 after retry RPC after error@2003105165) -> (continuation of Deferred@106400283 after retry RPC after error@1565404031) -> (continuation of Deferred@2139079521 after retry RPC after error@803707294) -> (continuation of Deferred@366659883 after retry RPC after error@962792764) -> (continuation of Deferred@948352442 after retry RPC after error@733933661) -> (continuation of Deferred@1760491408 after retry RPC after error@1229262172) -> (continuation of Deferred@1147357867 after retry RPC after error@411264610) -> (continuation of Deferred@122572949 after retry RPC after error@1711313613) -> (continuation of Deferred@946387636 after retry RPC after error@995851181) -> (continuation of Deferred@1323589291 after retry RPC after error@739257152) -> (continuation of Deferred@1372625260 after retry RPC after error@825628392) -> (continuation of Deferred@1573590883 after retry RPC after error@1890352561) -> (continuation of Deferred@635605297 after retry RPC after error@610419251) -> (continuation of Deferred@555170259 after retry RPC after error@843418037) -> (continuation of Deferred@52428245 after retry RPC after error@895112287) -> (continuation of Deferred@756755078 after retry RPC after error@856495793) -> (continuation of Deferred@1325442128 after retry RPC after error@673184067) -> (continuation of Deferred@1545910748 after retry RPC after error@122127479) -> (continuation of Deferred@1294959339 after retry RPC after error@447594476) -> (continuation of Deferred@1673400124 after retry RPC after error@1491159808) -> (continuation of Deferred@1776857645 after retry RPC after error@516760099) -> (continuation of Deferred@219748649 after retry RPC after error@1601370817) -> (continuation of Deferred@1940345560 after retry RPC after error@76327605), errback=(continuation of Deferred@1607012979 after retry RPC after error@100652726) -> (continuation of Deferred@370998217 after retry RPC after error@462953309) -> (continuation of Deferred@206952663 after retry RPC after error@2003105165) -> (continuation of Deferred@106400283 after retry RPC after error@1565404031) -> (continuation of Deferred@2139079521 after retry RPC after error@803707294) -> (continuation of Deferred@366659883 after retry RPC after error@962792764) -> (continuation of Deferred@948352442 after retry RPC after error@733933661) -> (continuation of Deferred@1760491408 after retry RPC after error@1229262172) -> (continuation of Deferred@1147357867 after retry RPC after error@411264610) -> (continuation of Deferred@122572949 after retry RPC after error@1711313613) -> (continuation of Deferred@946387636 after retry RPC after error@995851181) -> (continuation of Deferred@1323589291 after retry RPC after error@739257152) -> (continuation of Deferred@1372625260 after retry RPC after error@825628392) -> (continuation of Deferred@1573590883 after retry RPC after error@1890352561) -> (continuation of Deferred@635605297 after retry RPC after error@610419251) -> (continuation of Deferred@555170259 after retry RPC after error@843418037) -> (continuation of Deferred@52428245 after retry RPC after error@895112287) -> (continuation of Deferred@756755078 after retry RPC after error@856495793) -> (continuation of Deferred@1325442128 after retry RPC after error@673184067) -> (continuation of Deferred@1545910748 after retry RPC after error@122127479) -> (continuation of Deferred@1294959339 after retry RPC after error@447594476) -> (continuation of Deferred@1673400124 after retry RPC after error@1491159808) -> (continuation of Deferred@1776857645 after retry RPC after error@516760099) -> (continuation of Deferred@219748649 after retry RPC after error@1601370817) -> (continuation of Deferred@1940345560 after retry RPC after error@76327605)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.3.253.62:46169
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
11:54:42.699 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
11:54:42.699 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=26906), Traces: [0ms] refreshing cache from master, [36ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.60:33837, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.62:46169, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.61:46581, [190ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.62:46169: Network error: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169, [254ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.61:46581: Network error: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [541ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.60:33837: OK, [545ms] delaying RPC due to: Service unavailable: Master config (127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [556ms] refreshing cache from master, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.60:33837, [558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.62:46169, [560ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.61:46581, [562ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.62:46169: Network error: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169, [564ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.60:33837: OK, [566ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.61:46581: Network error: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [567ms] delaying RPC due to: Service unavailable: Master config (127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [576ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.60:33837, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.62:46169, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.61:46581, [582ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.62:46169: Network error: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169, [583ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.60:33837: OK, [585ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.61:46581: Network error: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [586ms] delaying RPC due to: Service unavailable: Master config (127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [596ms] refreshing cache from master, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.60:33837, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.62:46169, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.61:46581, [601ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.62:46169: Network error: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169, [602ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.60:33837: OK, [603ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.61:46581: Network error: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [605ms] delaying RPC due to: Service unavailable: Master config (127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [636ms] refreshing cache from master, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.60:33837, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.62:46169, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.61:46581, [640ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.62:46169: Network error: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169, [644ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.60:33837: OK, [645ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.61:46581: Network error: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [646ms] delaying RPC due to: Service unavailable: Master config (127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [676ms] refreshing cache from master, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.60:33837, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.62:46169, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.61:46581, [680ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.62:46169: Network error: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169, [683ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.60:33837: OK, [683ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.61:46581: Network error: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [685ms] delaying RPC due to: Service unavailable: Master config (127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [736ms] refreshing cache from master, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.60:33837, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.62:46169, [739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.61:46581, [742ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.62:46169: Network error: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169, [742ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.60:33837: OK, [743ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.61:46581: Network error: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [744ms] delaying RPC due to: Service unavailable: Master config (127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [876ms] refreshing cache from master, [876ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.60:33837, [876ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.62:46169, [878ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.61:46581, [880ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.62:46169: Network error: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169, [880ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.60:33837: OK, [884ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.61:46581: Network error: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [885ms] delaying RPC due to: Service unavailable: Master config (127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [1116ms] refreshing cache from master, [1116ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.60:33837, [1117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.62:46169, [1118ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.61:46581, [1122ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.62:46169: Network error: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169, [1123ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.60:33837: OK, [1124ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.61:46581: Network error: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [1125ms] delaying RPC due to: Service unavailable: Master config (127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [1576ms] refreshing cache from master, [1576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.60:33837, [1577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.62:46169, [1578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.61:46581, [1580ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.62:46169: Network error: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169, [1582ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.61:46581: Network error: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [1583ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.60:33837: OK, [1584ms] delaying RPC due to: Service unavailable: Master config (127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [1676ms] refreshing cache from master, [1676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.60:33837, [1676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.62:46169, [1677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.61:46581, [1679ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.62:46169: Network error: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169, [1680ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.60:33837: OK, [1681ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.61:46581: Network error: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [1682ms] delaying RPC due to: Service unavailable: Master config (127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [2216ms] refreshing cache from master, [2216ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.60:33837, [2216ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.62:46169, [2217ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.61:46581, [2219ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.62:46169: Network error: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169, [2221ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.60:33837: OK, [2221ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.61:46581: Network error: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [2221ms] delaying RPC due to: Service unavailable: Master config (127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [3416ms] refreshing cache from master, [3416ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.60:33837, [3416ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.62:46169, [3418ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.61:46581, [3420ms] too many traces: truncated at 100 traces, deferred=Deferred@760482748(state=PENDING, result=null, callback=(continuation of Deferred@1607012979 after retry RPC after error@100652726) -> (continuation of Deferred@370998217 after retry RPC after error@462953309) -> (continuation of Deferred@206952663 after retry RPC after error@2003105165) -> (continuation of Deferred@106400283 after retry RPC after error@1565404031) -> (continuation of Deferred@2139079521 after retry RPC after error@803707294) -> (continuation of Deferred@366659883 after retry RPC after error@962792764) -> (continuation of Deferred@948352442 after retry RPC after error@733933661) -> (continuation of Deferred@1760491408 after retry RPC after error@1229262172) -> (continuation of Deferred@1147357867 after retry RPC after error@411264610) -> (continuation of Deferred@122572949 after retry RPC after error@1711313613) -> (continuation of Deferred@946387636 after retry RPC after error@995851181) -> (continuation of Deferred@1323589291 after retry RPC after error@739257152) -> (continuation of Deferred@1372625260 after retry RPC after error@825628392) -> (continuation of Deferred@1573590883 after retry RPC after error@1890352561) -> (continuation of Deferred@635605297 after retry RPC after error@610419251) -> (continuation of Deferred@555170259 after retry RPC after error@843418037) -> (continuation of Deferred@52428245 after retry RPC after error@895112287) -> (continuation of Deferred@756755078 after retry RPC after error@856495793) -> (continuation of Deferred@1325442128 after retry RPC after error@673184067) -> (continuation of Deferred@1545910748 after retry RPC after error@122127479) -> (continuation of Deferred@1294959339 after retry RPC after error@447594476) -> (continuation of Deferred@1673400124 after retry RPC after error@1491159808) -> (continuation of Deferred@1776857645 after retry RPC after error@516760099) -> (continuation of Deferred@219748649 after retry RPC after error@1601370817) -> (continuation of Deferred@1940345560 after retry RPC after error@76327605), errback=(continuation of Deferred@1607012979 after retry RPC after error@100652726) -> (continuation of Deferred@370998217 after retry RPC after error@462953309) -> (continuation of Deferred@206952663 after retry RPC after error@2003105165) -> (continuation of Deferred@106400283 after retry RPC after error@1565404031) -> (continuation of Deferred@2139079521 after retry RPC after error@803707294) -> (continuation of Deferred@366659883 after retry RPC after error@962792764) -> (continuation of Deferred@948352442 after retry RPC after error@733933661) -> (continuation of Deferred@1760491408 after retry RPC after error@1229262172) -> (continuation of Deferred@1147357867 after retry RPC after error@411264610) -> (continuation of Deferred@122572949 after retry RPC after error@1711313613) -> (continuation of Deferred@946387636 after retry RPC after error@995851181) -> (continuation of Deferred@1323589291 after retry RPC after error@739257152) -> (continuation of Deferred@1372625260 after retry RPC after error@825628392) -> (continuation of Deferred@1573590883 after retry RPC after error@1890352561) -> (continuation of Deferred@635605297 after retry RPC after error@610419251) -> (continuation of Deferred@555170259 after retry RPC after error@843418037) -> (continuation of Deferred@52428245 after retry RPC after error@895112287) -> (continuation of Deferred@756755078 after retry RPC after error@856495793) -> (continuation of Deferred@1325442128 after retry RPC after error@673184067) -> (continuation of Deferred@1545910748 after retry RPC after error@122127479) -> (continuation of Deferred@1294959339 after retry RPC after error@447594476) -> (continuation of Deferred@1673400124 after retry RPC after error@1491159808) -> (continuation of Deferred@1776857645 after retry RPC after error@516760099) -> (continuation of Deferred@219748649 after retry RPC after error@1601370817) -> (continuation of Deferred@1940345560 after retry RPC after error@76327605)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.3.253.62:46169
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=26906), Traces: [0ms] refreshing cache from master, [36ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.60:33837, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.62:46169, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.61:46581, [190ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.62:46169: Network error: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169, [254ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.61:46581: Network error: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [541ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.60:33837: OK, [545ms] delaying RPC due to: Service unavailable: Master config (127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [556ms] refreshing cache from master, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.60:33837, [558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.62:46169, [560ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.61:46581, [562ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.62:46169: Network error: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169, [564ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.60:33837: OK, [566ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.61:46581: Network error: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [567ms] delaying RPC due to: Service unavailable: Master config (127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [576ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.60:33837, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.62:46169, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.61:46581, [582ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.62:46169: Network error: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169, [583ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.60:33837: OK, [585ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.61:46581: Network error: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [586ms] delaying RPC due to: Service unavailable: Master config (127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [596ms] refreshing cache from master, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.60:33837, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.62:46169, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.61:46581, [601ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.62:46169: Network error: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169, [602ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.60:33837: OK, [603ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.61:46581: Network error: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [605ms] delaying RPC due to: Service unavailable: Master config (127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [636ms] refreshing cache from master, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.60:33837, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.62:46169, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.61:46581, [640ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.62:46169: Network error: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169, [644ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.60:33837: OK, [645ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.61:46581: Network error: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [646ms] delaying RPC due to: Service unavailable: Master config (127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [676ms] refreshing cache from master, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.60:33837, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.62:46169, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.61:46581, [680ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.62:46169: Network error: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169, [683ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.60:33837: OK, [683ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.61:46581: Network error: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [685ms] delaying RPC due to: Service unavailable: Master config (127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [736ms] refreshing cache from master, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.60:33837, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.62:46169, [739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.61:46581, [742ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.62:46169: Network error: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169, [742ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.60:33837: OK, [743ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.61:46581: Network error: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [744ms] delaying RPC due to: Service unavailable: Master config (127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [876ms] refreshing cache from master, [876ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.60:33837, [876ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.62:46169, [878ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.61:46581, [880ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.62:46169: Network error: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169, [880ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.60:33837: OK, [884ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.61:46581: Network error: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [885ms] delaying RPC due to: Service unavailable: Master config (127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [1116ms] refreshing cache from master, [1116ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.60:33837, [1117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.62:46169, [1118ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.61:46581, [1122ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.62:46169: Network error: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169, [1123ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.60:33837: OK, [1124ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.61:46581: Network error: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [1125ms] delaying RPC due to: Service unavailable: Master config (127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [1576ms] refreshing cache from master, [1576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.60:33837, [1577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.62:46169, [1578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.61:46581, [1580ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.62:46169: Network error: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169, [1582ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.61:46581: Network error: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [1583ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.60:33837: OK, [1584ms] delaying RPC due to: Service unavailable: Master config (127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [1676ms] refreshing cache from master, [1676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.60:33837, [1676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.62:46169, [1677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.61:46581, [1679ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.62:46169: Network error: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169, [1680ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.60:33837: OK, [1681ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.61:46581: Network error: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [1682ms] delaying RPC due to: Service unavailable: Master config (127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [2216ms] refreshing cache from master, [2216ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.60:33837, [2216ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.62:46169, [2217ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.61:46581, [2219ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.62:46169: Network error: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169, [2221ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.60:33837: OK, [2221ms] Sub RPC ConnectToMaster: received response from server master-127.3.253.61:46581: Network error: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [2221ms] delaying RPC due to: Service unavailable: Master config (127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581, [3416ms] refreshing cache from master, [3416ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.60:33837, [3416ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.62:46169, [3418ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.253.61:46581, [3420ms] too many traces: truncated at 100 traces, deferred=Deferred@760482748(state=PENDING, result=null, callback=(continuation of Deferred@1607012979 after retry RPC after error@100652726) -> (continuation of Deferred@370998217 after retry RPC after error@462953309) -> (continuation of Deferred@206952663 after retry RPC after error@2003105165) -> (continuation of Deferred@106400283 after retry RPC after error@1565404031) -> (continuation of Deferred@2139079521 after retry RPC after error@803707294) -> (continuation of Deferred@366659883 after retry RPC after error@962792764) -> (continuation of Deferred@948352442 after retry RPC after error@733933661) -> (continuation of Deferred@1760491408 after retry RPC after error@1229262172) -> (continuation of Deferred@1147357867 after retry RPC after error@411264610) -> (continuation of Deferred@122572949 after retry RPC after error@1711313613) -> (continuation of Deferred@946387636 after retry RPC after error@995851181) -> (continuation of Deferred@1323589291 after retry RPC after error@739257152) -> (continuation of Deferred@1372625260 after retry RPC after error@825628392) -> (continuation of Deferred@1573590883 after retry RPC after error@1890352561) -> (continuation of Deferred@635605297 after retry RPC after error@610419251) -> (continuation of Deferred@555170259 after retry RPC after error@843418037) -> (continuation of Deferred@52428245 after retry RPC after error@895112287) -> (continuation of Deferred@756755078 after retry RPC after error@856495793) -> (continuation of Deferred@1325442128 after retry RPC after error@673184067) -> (continuation of Deferred@1545910748 after retry RPC after error@122127479) -> (continuation of Deferred@1294959339 after retry RPC after error@447594476) -> (continuation of Deferred@1673400124 after retry RPC after error@1491159808) -> (continuation of Deferred@1776857645 after retry RPC after error@516760099) -> (continuation of Deferred@219748649 after retry RPC after error@1601370817) -> (continuation of Deferred@1940345560 after retry RPC after error@76327605), errback=(continuation of Deferred@1607012979 after retry RPC after error@100652726) -> (continuation of Deferred@370998217 after retry RPC after error@462953309) -> (continuation of Deferred@206952663 after retry RPC after error@2003105165) -> (continuation of Deferred@106400283 after retry RPC after error@1565404031) -> (continuation of Deferred@2139079521 after retry RPC after error@803707294) -> (continuation of Deferred@366659883 after retry RPC after error@962792764) -> (continuation of Deferred@948352442 after retry RPC after error@733933661) -> (continuation of Deferred@1760491408 after retry RPC after error@1229262172) -> (continuation of Deferred@1147357867 after retry RPC after error@411264610) -> (continuation of Deferred@122572949 after retry RPC after error@1711313613) -> (continuation of Deferred@946387636 after retry RPC after error@995851181) -> (continuation of Deferred@1323589291 after retry RPC after error@739257152) -> (continuation of Deferred@1372625260 after retry RPC after error@825628392) -> (continuation of Deferred@1573590883 after retry RPC after error@1890352561) -> (continuation of Deferred@635605297 after retry RPC after error@610419251) -> (continuation of Deferred@555170259 after retry RPC after error@843418037) -> (continuation of Deferred@52428245 after retry RPC after error@895112287) -> (continuation of Deferred@756755078 after retry RPC after error@856495793) -> (continuation of Deferred@1325442128 after retry RPC after error@673184067) -> (continuation of Deferred@1545910748 after retry RPC after error@122127479) -> (continuation of Deferred@1294959339 after retry RPC after error@447594476) -> (continuation of Deferred@1673400124 after retry RPC after error@1491159808) -> (continuation of Deferred@1776857645 after retry RPC after error@516760099) -> (continuation of Deferred@219748649 after retry RPC after error@1601370817) -> (continuation of Deferred@1940345560 after retry RPC after error@76327605)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.3.253.60:33837,127.3.253.62:46169,127.3.253.61:46581) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.61:46581(127.3.253.61:46581): Connection refused: /127.3.253.61:46581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.253.62:46169(127.3.253.62:46169): Connection refused: /127.3.253.62:46169
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.3.253.62:46169
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)