03:46:04.442 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
03:46:04.454 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskbYfj1w/test-tmp
03:46:04.456 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
03:46:04.460 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskbYfj1w/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
03:46:04.661 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
03:46:04.833 [DEBUG - main] (MiniKuduCluster.java:195) Response:
03:46:04.834 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
03:46:04.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:04.837917 3758 mini_oidc.cc:152] Starting JWKS server
03:46:04.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:04.841922 3758 webserver.cc:460] Webserver started at http://127.0.0.1:33563/ using document root <none> and password file <none>
03:46:04.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:04.842387 3758 mini_oidc.cc:189] Starting OIDC Discovery server
03:46:04.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:04.844116 3758 webserver.cc:460] Webserver started at http://127.0.0.1:35323/ using document root <none> and password file <none>
03:46:04.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-28T03:46:04Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
03:46:04.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-28T03:46:04Z Disabled control of system clock
03:46:04.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:04.893512 3758 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskbYfj1w/build/tsan/bin/kudu
03:46:04.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbYfj1w/build/tsan/bin/kudu
03:46:04.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/master-0/wal
03:46:04.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/master-0/data
03:46:04.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:46:04.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:46:04.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:46:04.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:46:04.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:46:04.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:46:04.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/master-0/logs
03:46:04.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/master-0/data/info.pb
03:46:04.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:46:04.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:46:04.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:46:04.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:46:04.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:46:04.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:46:04.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:46:04.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:46:04.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:46:04.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:46:04.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:46:04.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:46:04.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.171.190:36307
03:46:04.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.171.190
03:46:04.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:46:04.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.171.148:45923
03:46:04.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:46:04.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:46:04.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:46:04.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:46:04.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.171.190:36307,127.3.171.189:33249,127.3.171.188:38157
03:46:04.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:46:04.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33563/jwks.json with env {}
03:46:05.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:05.183861 3770 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:46:05.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:05.184391 3770 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:46:05.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:05.185048 3770 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:46:05.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:05.215670 3770 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:46:05.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:05.215947 3770 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:46:05.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:05.216135 3770 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:46:05.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:05.216398 3770 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:46:05.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:05.216688 3770 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:46:05.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:05.216940 3770 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33563/jwks.json
03:46:05.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:05.251338 3770 master_runner.cc:386] Master server non-default flags:
03:46:05.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:46:05.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.171.148:45923
03:46:05.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:46:05.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:46:05.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/master-0/data
03:46:05.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/master-0/wal
03:46:05.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:46:05.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.171.190:36307,127.3.171.189:33249,127.3.171.188:38157
03:46:05.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:46:05.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:46:05.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:46:05.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.171.190:36307
03:46:05.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:46:05.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:46:05.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:46:05.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33563/jwks.json
03:46:05.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:46:05.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:46:05.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/master-0/data/info.pb
03:46:05.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.171.190
03:46:05.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:46:05.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:46:05.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:46:05.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:46:05.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:46:05.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:46:05.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/master-0/logs
03:46:05.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:46:05.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:46:05.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:46:05.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:46:05.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:46:05.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:46:05.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:46:05.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 02:50:13 UTC on ccycloud.quasar-purgqw.root.comops.site
03:46:05.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676527
03:46:05.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:46:05.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:05.252725 3770 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:46:05.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:05.254755 3770 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:46:05.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:05.272511 3776 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:46:06.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:05.272883 3779 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:46:06.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:05.274020 3770 server_base.cc:1034] running on GCE node
03:46:06.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:05.272504 3777 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:46:06.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:06.461251 3770 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:46:06.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:06.463902 3770 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:46:06.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:06.465237 3770 hybrid_clock.cc:648] HybridClock initialized: now 1790567166465211 us; error 46 us; skew 500 ppm
03:46:06.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:06.466023 3770 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:46:06.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:06.472307 3770 webserver.cc:460] Webserver started at http://127.3.171.190:44493/ using document root <none> and password file <none>
03:46:06.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:06.473430 3770 fs_manager.cc:359] Metadata directory not provided
03:46:06.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:06.473695 3770 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:46:06.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:06.474287 3770 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:46:06.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:06.479619 3770 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/master-0/data/instance:
03:46:06.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "934dd201770e4f2b8cf9a0cc7ac7659a"
03:46:06.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 03:46:06 on dist-test-slave-n4rz"
03:46:06.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:06.481117 3770 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/master-0/wal/instance:
03:46:06.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "934dd201770e4f2b8cf9a0cc7ac7659a"
03:46:06.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 03:46:06 on dist-test-slave-n4rz"
03:46:06.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:06.488715 3770 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
03:46:06.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:06.494782 3786 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:46:06.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:06.495850 3770 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
03:46:06.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:06.496224 3770 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/master-0/data,/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/master-0/wal
03:46:06.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "934dd201770e4f2b8cf9a0cc7ac7659a"
03:46:06.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 03:46:06 on dist-test-slave-n4rz"
03:46:06.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:06.496656 3770 fs_report.cc:383] FS layout report
03:46:06.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:46:06.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/master-0/wal
03:46:06.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/master-0/wal
03:46:06.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/master-0/data/data
03:46:06.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:46:06.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:46:06.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:46:06.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:46:06.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:46:06.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:46:06.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:46:06.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:46:06.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:46:06.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:46:06.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:46:06.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:06.560009 3770 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:46:06.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:06.562101 3770 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:46:06.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:06.562709 3770 kserver.cc:163] Server-wide thread pool size limit: 3276
03:46:06.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:06.636610 3770 rpc_server.cc:307] RPC server started. Bound to: 127.3.171.190:36307
03:46:06.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:06.636722 3837 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.3.171.190:36307 every 8 connection(s)
03:46:06.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:06.639194 3770 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/master-0/data/info.pb
03:46:06.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:06.644270 3838 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:46:06.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:06.650287 3758 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskbYfj1w/build/tsan/bin/kudu as pid 3770
03:46:06.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:06.650864 3758 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/master-0/wal/instance
03:46:06.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:06.656818 3758 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskbYfj1w/build/tsan/bin/kudu
03:46:06.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbYfj1w/build/tsan/bin/kudu
03:46:06.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/master-1/wal
03:46:06.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/master-1/data
03:46:06.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:46:06.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:46:06.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:46:06.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:46:06.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:46:06.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:46:06.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/master-1/logs
03:46:06.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/master-1/data/info.pb
03:46:06.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:46:06.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:46:06.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:46:06.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:46:06.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:46:06.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:46:06.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:46:06.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:46:06.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:46:06.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:46:06.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:46:06.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:46:06.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.171.189:33249
03:46:06.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.171.189
03:46:06.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:46:06.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.171.148:45923
03:46:06.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:46:06.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:46:06.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:46:06.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:46:06.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.171.190:36307,127.3.171.189:33249,127.3.171.188:38157
03:46:06.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:46:06.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33563/jwks.json with env {}
03:46:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:06.660660 3838 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.171.190" port: 36307 } has no permanent_uuid. Determining permanent_uuid...
03:46:06.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:06.692829 3838 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.171.189" port: 33249 } has no permanent_uuid. Determining permanent_uuid...
03:46:06.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:06.695423 3790 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.171.189:33249: connect: Connection refused (error 111)
03:46:06.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:06.699277 3838 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.171.189:33249: Network error: Client connection negotiation failed: client connection to 127.3.171.189:33249: connect: Connection refused (error 111)
03:46:06.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:06.748766 3838 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.171.189" port: 33249 } attempt: 1
03:46:06.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:06.753096 3838 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.171.189:33249: Network error: Client connection negotiation failed: client connection to 127.3.171.189:33249: connect: Connection refused (error 111)
03:46:06.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:06.821478 3838 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.171.189" port: 33249 } attempt: 2
03:46:06.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:06.825155 3838 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.171.189:33249: Network error: Client connection negotiation failed: client connection to 127.3.171.189:33249: connect: Connection refused (error 111)
03:46:06.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:06.916553 3838 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.171.189" port: 33249 } attempt: 3
03:46:06.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:06.920116 3838 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.171.189:33249: Network error: Client connection negotiation failed: client connection to 127.3.171.189:33249: connect: Connection refused (error 111)
03:46:06.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:06.963934 3840 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:46:06.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:06.964486 3840 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:46:06.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:06.965137 3840 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:46:06.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:06.995476 3840 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:46:06.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:06.995775 3840 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:46:06.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:06.995968 3840 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:46:06.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:06.996146 3840 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:46:06.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:06.996320 3840 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:46:06.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:06.996477 3840 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33563/jwks.json
03:46:07.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:07.031423 3840 master_runner.cc:386] Master server non-default flags:
03:46:07.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:46:07.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.171.148:45923
03:46:07.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:46:07.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:46:07.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/master-1/data
03:46:07.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/master-1/wal
03:46:07.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:46:07.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.171.190:36307,127.3.171.189:33249,127.3.171.188:38157
03:46:07.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:46:07.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:46:07.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:46:07.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.171.189:33249
03:46:07.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:46:07.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:46:07.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:46:07.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33563/jwks.json
03:46:07.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:46:07.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:46:07.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/master-1/data/info.pb
03:46:07.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.171.189
03:46:07.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:46:07.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:46:07.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:46:07.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:46:07.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:46:07.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:46:07.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/master-1/logs
03:46:07.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:46:07.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:46:07.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:46:07.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:46:07.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:46:07.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:46:07.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:46:07.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 02:50:13 UTC on ccycloud.quasar-purgqw.root.comops.site
03:46:07.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676527
03:46:07.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:46:07.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:07.032624 3840 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:46:07.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:07.034705 3840 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:46:07.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:07.052315 3850 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:46:07.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:07.063558 3838 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.171.189" port: 33249 } attempt: 4
03:46:07.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:07.070240 3838 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.171.189:33249: Network error: Client connection negotiation failed: client connection to 127.3.171.189:33249: connect: Connection refused (error 111)
03:46:07.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:07.369760 3838 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.171.189" port: 33249 } attempt: 5
03:46:07.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:07.374807 3838 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.171.189:33249: Network error: Client connection negotiation failed: client connection to 127.3.171.189:33249: connect: Connection refused (error 111)
03:46:07.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:07.922199 3838 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.171.189" port: 33249 } attempt: 6
03:46:07.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:07.930948 3838 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.171.189:33249: Network error: Client connection negotiation failed: client connection to 127.3.171.189:33249: connect: Connection refused (error 111)
03:46:08.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:07.052346 3849 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:46:08.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:07.052574 3852 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:46:08.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:07.054867 3840 server_base.cc:1034] running on GCE node
03:46:08.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:08.235469 3840 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:46:08.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:08.238813 3840 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:46:08.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:08.240265 3840 hybrid_clock.cc:648] HybridClock initialized: now 1790567168240172 us; error 114 us; skew 500 ppm
03:46:08.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:08.241309 3840 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:46:08.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:08.251951 3840 webserver.cc:460] Webserver started at http://127.3.171.189:43357/ using document root <none> and password file <none>
03:46:08.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:08.253042 3840 fs_manager.cc:359] Metadata directory not provided
03:46:08.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:08.253265 3840 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:46:08.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:08.253762 3840 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:46:08.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:08.258937 3840 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/master-1/data/instance:
03:46:08.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a579ba8360cd48569bdc7938e46df544"
03:46:08.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 03:46:08 on dist-test-slave-n4rz"
03:46:08.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:08.260262 3840 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/master-1/wal/instance:
03:46:08.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a579ba8360cd48569bdc7938e46df544"
03:46:08.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 03:46:08 on dist-test-slave-n4rz"
03:46:08.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:08.268069 3840 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.000s
03:46:08.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:08.273741 3861 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:46:08.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:08.274816 3840 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.000s sys 0.004s
03:46:08.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:08.275134 3840 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/master-1/data,/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/master-1/wal
03:46:08.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a579ba8360cd48569bdc7938e46df544"
03:46:08.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 03:46:08 on dist-test-slave-n4rz"
03:46:08.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:08.275451 3840 fs_report.cc:383] FS layout report
03:46:08.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:46:08.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/master-1/wal
03:46:08.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/master-1/wal
03:46:08.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/master-1/data/data
03:46:08.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:46:08.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:46:08.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:46:08.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:46:08.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:46:08.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:46:08.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:46:08.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:46:08.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:46:08.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:46:08.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:46:08.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:08.334143 3840 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:46:08.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:08.335688 3840 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:46:08.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:08.336133 3840 kserver.cc:163] Server-wide thread pool size limit: 3276
03:46:08.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:08.408973 3840 rpc_server.cc:307] RPC server started. Bound to: 127.3.171.189:33249
03:46:08.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:08.409121 3912 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.3.171.189:33249 every 8 connection(s)
03:46:08.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:08.411764 3840 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/master-1/data/info.pb
03:46:08.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:08.416759 3913 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:46:08.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:08.419735 3758 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskbYfj1w/build/tsan/bin/kudu as pid 3840
03:46:08.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:08.420171 3758 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/master-1/wal/instance
03:46:08.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:08.422194 3758 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskbYfj1w/build/tsan/bin/kudu
03:46:08.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbYfj1w/build/tsan/bin/kudu
03:46:08.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/master-2/wal
03:46:08.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/master-2/data
03:46:08.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:46:08.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:46:08.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:46:08.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:46:08.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:46:08.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:46:08.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/master-2/logs
03:46:08.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/master-2/data/info.pb
03:46:08.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:46:08.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:46:08.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:46:08.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:46:08.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:46:08.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:46:08.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:46:08.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:46:08.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:46:08.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:46:08.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:46:08.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:46:08.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.171.188:38157
03:46:08.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.171.188
03:46:08.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:46:08.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.171.148:45923
03:46:08.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:46:08.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:46:08.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:46:08.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:46:08.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.171.190:36307,127.3.171.189:33249,127.3.171.188:38157
03:46:08.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:46:08.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33563/jwks.json with env {}
03:46:08.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:08.428763 3913 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.171.190" port: 36307 } has no permanent_uuid. Determining permanent_uuid...
03:46:08.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:08.452491 3913 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.171.189" port: 33249 } has no permanent_uuid. Determining permanent_uuid...
03:46:08.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:08.466792 3913 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.171.188" port: 38157 } has no permanent_uuid. Determining permanent_uuid...
03:46:08.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:08.468940 3862 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.171.188:38157: connect: Connection refused (error 111)
03:46:08.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:08.472764 3913 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.171.188:38157: Network error: Client connection negotiation failed: client connection to 127.3.171.188:38157: connect: Connection refused (error 111)
03:46:08.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:08.522229 3913 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.171.188" port: 38157 } attempt: 1
03:46:08.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:08.526170 3913 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.171.188:38157: Network error: Client connection negotiation failed: client connection to 127.3.171.188:38157: connect: Connection refused (error 111)
03:46:08.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:08.594545 3913 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.171.188" port: 38157 } attempt: 2
03:46:08.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:08.598681 3913 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.171.188:38157: Network error: Client connection negotiation failed: client connection to 127.3.171.188:38157: connect: Connection refused (error 111)
03:46:08.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:08.690126 3913 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.171.188" port: 38157 } attempt: 3
03:46:08.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:08.693518 3913 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.171.188:38157: Network error: Client connection negotiation failed: client connection to 127.3.171.188:38157: connect: Connection refused (error 111)
03:46:08.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:08.722038 3915 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:46:08.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:08.722558 3915 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:46:08.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:08.722976 3915 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:46:08.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:08.753543 3915 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:46:08.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:08.753803 3915 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:46:08.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:08.753978 3915 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:46:08.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:08.754266 3915 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:46:08.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:08.754484 3915 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:46:08.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:08.754683 3915 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33563/jwks.json
03:46:08.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:08.788378 3915 master_runner.cc:386] Master server non-default flags:
03:46:08.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:46:08.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.171.148:45923
03:46:08.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:46:08.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:46:08.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/master-2/data
03:46:08.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/master-2/wal
03:46:08.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:46:08.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.171.190:36307,127.3.171.189:33249,127.3.171.188:38157
03:46:08.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:46:08.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:46:08.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:46:08.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.171.188:38157
03:46:08.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:46:08.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:46:08.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:46:08.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33563/jwks.json
03:46:08.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:46:08.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:46:08.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/master-2/data/info.pb
03:46:08.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.171.188
03:46:08.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:46:08.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:46:08.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:46:08.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:46:08.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:46:08.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:46:08.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/master-2/logs
03:46:08.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:46:08.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:46:08.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:46:08.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:46:08.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:46:08.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:46:08.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:46:08.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 02:50:13 UTC on ccycloud.quasar-purgqw.root.comops.site
03:46:08.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676527
03:46:08.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:46:08.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:08.789597 3915 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:46:08.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:08.791313 3915 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:46:08.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:08.808706 3925 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:46:08.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:08.836915 3913 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.171.188" port: 38157 } attempt: 4
03:46:08.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:08.840651 3913 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.171.188:38157: Network error: Client connection negotiation failed: client connection to 127.3.171.188:38157: connect: Connection refused (error 111)
03:46:08.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:08.991330 3838 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.171.189" port: 33249 } attempt: 7
03:46:09.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:09.018828 3838 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.171.188" port: 38157 } has no permanent_uuid. Determining permanent_uuid...
03:46:09.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:09.024233 3838 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.171.188:38157: Network error: Client connection negotiation failed: client connection to 127.3.171.188:38157: connect: Connection refused (error 111)
03:46:09.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:09.082919 3838 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.171.188" port: 38157 } attempt: 1
03:46:09.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:09.087939 3838 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.171.188:38157: Network error: Client connection negotiation failed: client connection to 127.3.171.188:38157: connect: Connection refused (error 111)
03:46:09.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:09.140264 3913 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.171.188" port: 38157 } attempt: 5
03:46:09.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:09.144456 3913 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.171.188:38157: Network error: Client connection negotiation failed: client connection to 127.3.171.188:38157: connect: Connection refused (error 111)
03:46:09.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:09.169390 3838 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.171.188" port: 38157 } attempt: 2
03:46:09.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:09.173843 3838 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.171.188:38157: Network error: Client connection negotiation failed: client connection to 127.3.171.188:38157: connect: Connection refused (error 111)
03:46:09.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:09.259208 3838 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.171.188" port: 38157 } attempt: 3
03:46:09.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:09.263609 3838 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.171.188:38157: Network error: Client connection negotiation failed: client connection to 127.3.171.188:38157: connect: Connection refused (error 111)
03:46:09.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:09.403980 3838 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.171.188" port: 38157 } attempt: 4
03:46:09.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:09.407621 3838 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.171.188:38157: Network error: Client connection negotiation failed: client connection to 127.3.171.188:38157: connect: Connection refused (error 111)
03:46:09.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:09.691022 3838 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.171.188" port: 38157 } attempt: 5
03:46:09.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:09.692029 3913 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.171.188" port: 38157 } attempt: 6
03:46:09.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:09.696969 3838 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.171.188:38157: Network error: Client connection negotiation failed: client connection to 127.3.171.188:38157: connect: Connection refused (error 111)
03:46:09.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:09.700057 3913 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.171.188:38157: Network error: Client connection negotiation failed: client connection to 127.3.171.188:38157: connect: Connection refused (error 111)
03:46:10.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:08.808768 3926 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:46:10.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:08.810541 3915 server_base.cc:1034] running on GCE node
03:46:10.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:08.809233 3928 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:46:10.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.018209 3915 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:46:10.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.021217 3915 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:46:10.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.022634 3915 hybrid_clock.cc:648] HybridClock initialized: now 1790567170022575 us; error 78 us; skew 500 ppm
03:46:10.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.023437 3915 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:46:10.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.029613 3915 webserver.cc:460] Webserver started at http://127.3.171.188:43679/ using document root <none> and password file <none>
03:46:10.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.030493 3915 fs_manager.cc:359] Metadata directory not provided
03:46:10.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.030680 3915 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:46:10.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.031047 3915 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:46:10.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.037819 3915 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/master-2/data/instance:
03:46:10.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "69844543993b4308a9c886a9fff341ca"
03:46:10.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 03:46:10 on dist-test-slave-n4rz"
03:46:10.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.039297 3915 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/master-2/wal/instance:
03:46:10.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "69844543993b4308a9c886a9fff341ca"
03:46:10.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 03:46:10 on dist-test-slave-n4rz"
03:46:10.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.048329 3915 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.006s sys 0.005s
03:46:10.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.054059 3940 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:46:10.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.055110 3915 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
03:46:10.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.055441 3915 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/master-2/data,/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/master-2/wal
03:46:10.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "69844543993b4308a9c886a9fff341ca"
03:46:10.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 03:46:10 on dist-test-slave-n4rz"
03:46:10.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.055882 3915 fs_report.cc:383] FS layout report
03:46:10.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:46:10.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/master-2/wal
03:46:10.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/master-2/wal
03:46:10.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/master-2/data/data
03:46:10.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:46:10.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:46:10.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:46:10.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:46:10.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:46:10.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:46:10.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:46:10.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:46:10.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:46:10.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:46:10.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:46:10.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.102032 3915 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:46:10.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.103605 3915 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:46:10.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.104120 3915 kserver.cc:163] Server-wide thread pool size limit: 3276
03:46:10.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.170261 3915 rpc_server.cc:307] RPC server started. Bound to: 127.3.171.188:38157
03:46:10.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.170325 3991 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.3.171.188:38157 every 8 connection(s)
03:46:10.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.173038 3915 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/master-2/data/info.pb
03:46:10.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.178120 3992 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:46:10.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.180693 3758 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskbYfj1w/build/tsan/bin/kudu as pid 3915
03:46:10.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.181119 3758 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/master-2/wal/instance
03:46:10.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.194460 3992 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.171.190" port: 36307 } has no permanent_uuid. Determining permanent_uuid...
03:46:10.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.221457 3992 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.171.189" port: 33249 } has no permanent_uuid. Determining permanent_uuid...
03:46:10.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.234558 3992 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.171.188" port: 38157 } has no permanent_uuid. Determining permanent_uuid...
03:46:10.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.249396 3838 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.171.188" port: 38157 } attempt: 6
03:46:10.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.251966 3992 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 69844543993b4308a9c886a9fff341ca: Bootstrap starting.
03:46:10.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.258975 3992 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 69844543993b4308a9c886a9fff341ca: Neither blocks nor log segments found. Creating new log.
03:46:10.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.260842 3992 log.cc:824] T 00000000000000000000000000000000 P 69844543993b4308a9c886a9fff341ca: Log is configured to *not* fsync() on all Append() calls
03:46:10.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.266995 3992 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 69844543993b4308a9c886a9fff341ca: No bootstrap required, opened a new log
03:46:10.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.270998 3838 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 934dd201770e4f2b8cf9a0cc7ac7659a: Bootstrap starting.
03:46:10.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.277771 3838 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 934dd201770e4f2b8cf9a0cc7ac7659a: Neither blocks nor log segments found. Creating new log.
03:46:10.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.280233 3838 log.cc:824] T 00000000000000000000000000000000 P 934dd201770e4f2b8cf9a0cc7ac7659a: Log is configured to *not* fsync() on all Append() calls
03:46:10.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.285923 3838 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 934dd201770e4f2b8cf9a0cc7ac7659a: No bootstrap required, opened a new log
03:46:10.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.294226 3992 raft_consensus.cc:348] T 00000000000000000000000000000000 P 69844543993b4308a9c886a9fff341ca [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "934dd201770e4f2b8cf9a0cc7ac7659a" member_type: VOTER last_known_addr { host: "127.3.171.190" port: 36307 } } peers { permanent_uuid: "a579ba8360cd48569bdc7938e46df544" member_type: VOTER last_known_addr { host: "127.3.171.189" port: 33249 } } peers { permanent_uuid: "69844543993b4308a9c886a9fff341ca" member_type: VOTER last_known_addr { host: "127.3.171.188" port: 38157 } }
03:46:10.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.294853 3992 raft_consensus.cc:374] T 00000000000000000000000000000000 P 69844543993b4308a9c886a9fff341ca [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:46:10.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.295156 3992 raft_consensus.cc:729] T 00000000000000000000000000000000 P 69844543993b4308a9c886a9fff341ca [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 69844543993b4308a9c886a9fff341ca, State: Initialized, Role: FOLLOWER
03:46:10.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.296388 3992 consensus_queue.cc:260] T 00000000000000000000000000000000 P 69844543993b4308a9c886a9fff341ca [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "934dd201770e4f2b8cf9a0cc7ac7659a" member_type: VOTER last_known_addr { host: "127.3.171.190" port: 36307 } } peers { permanent_uuid: "a579ba8360cd48569bdc7938e46df544" member_type: VOTER last_known_addr { host: "127.3.171.189" port: 33249 } } peers { permanent_uuid: "69844543993b4308a9c886a9fff341ca" member_type: VOTER last_known_addr { host: "127.3.171.188" port: 38157 } }
03:46:10.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.299398 4003 sys_catalog.cc:455] T 00000000000000000000000000000000 P 69844543993b4308a9c886a9fff341ca [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "934dd201770e4f2b8cf9a0cc7ac7659a" member_type: VOTER last_known_addr { host: "127.3.171.190" port: 36307 } } peers { permanent_uuid: "a579ba8360cd48569bdc7938e46df544" member_type: VOTER last_known_addr { host: "127.3.171.189" port: 33249 } } peers { permanent_uuid: "69844543993b4308a9c886a9fff341ca" member_type: VOTER last_known_addr { host: "127.3.171.188" port: 38157 } } }
03:46:10.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.300320 4003 sys_catalog.cc:458] T 00000000000000000000000000000000 P 69844543993b4308a9c886a9fff341ca [sys.catalog]: This master's current role is: FOLLOWER
03:46:10.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.300717 3992 sys_catalog.cc:564] T 00000000000000000000000000000000 P 69844543993b4308a9c886a9fff341ca [sys.catalog]: configured and running, proceeding with master startup.
03:46:10.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.310140 3838 raft_consensus.cc:348] T 00000000000000000000000000000000 P 934dd201770e4f2b8cf9a0cc7ac7659a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "934dd201770e4f2b8cf9a0cc7ac7659a" member_type: VOTER last_known_addr { host: "127.3.171.190" port: 36307 } } peers { permanent_uuid: "a579ba8360cd48569bdc7938e46df544" member_type: VOTER last_known_addr { host: "127.3.171.189" port: 33249 } } peers { permanent_uuid: "69844543993b4308a9c886a9fff341ca" member_type: VOTER last_known_addr { host: "127.3.171.188" port: 38157 } }
03:46:10.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.310783 3838 raft_consensus.cc:374] T 00000000000000000000000000000000 P 934dd201770e4f2b8cf9a0cc7ac7659a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:46:10.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.311048 3838 raft_consensus.cc:729] T 00000000000000000000000000000000 P 934dd201770e4f2b8cf9a0cc7ac7659a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 934dd201770e4f2b8cf9a0cc7ac7659a, State: Initialized, Role: FOLLOWER
03:46:10.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.312013 3838 consensus_queue.cc:260] T 00000000000000000000000000000000 P 934dd201770e4f2b8cf9a0cc7ac7659a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "934dd201770e4f2b8cf9a0cc7ac7659a" member_type: VOTER last_known_addr { host: "127.3.171.190" port: 36307 } } peers { permanent_uuid: "a579ba8360cd48569bdc7938e46df544" member_type: VOTER last_known_addr { host: "127.3.171.189" port: 33249 } } peers { permanent_uuid: "69844543993b4308a9c886a9fff341ca" member_type: VOTER last_known_addr { host: "127.3.171.188" port: 38157 } }
03:46:10.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.314832 4009 sys_catalog.cc:455] T 00000000000000000000000000000000 P 934dd201770e4f2b8cf9a0cc7ac7659a [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "934dd201770e4f2b8cf9a0cc7ac7659a" member_type: VOTER last_known_addr { host: "127.3.171.190" port: 36307 } } peers { permanent_uuid: "a579ba8360cd48569bdc7938e46df544" member_type: VOTER last_known_addr { host: "127.3.171.189" port: 33249 } } peers { permanent_uuid: "69844543993b4308a9c886a9fff341ca" member_type: VOTER last_known_addr { host: "127.3.171.188" port: 38157 } } }
03:46:10.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.316782 3838 sys_catalog.cc:564] T 00000000000000000000000000000000 P 934dd201770e4f2b8cf9a0cc7ac7659a [sys.catalog]: configured and running, proceeding with master startup.
03:46:10.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.317094 4009 sys_catalog.cc:458] T 00000000000000000000000000000000 P 934dd201770e4f2b8cf9a0cc7ac7659a [sys.catalog]: This master's current role is: FOLLOWER
03:46:10.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:10.323534 3792 tablet.cc:2369] T 00000000000000000000000000000000 P 934dd201770e4f2b8cf9a0cc7ac7659a: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:46:10.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:10.333498 4018 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 69844543993b4308a9c886a9fff341ca: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:46:10.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:10.334136 4018 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:46:10.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:10.359063 3946 tablet.cc:2369] T 00000000000000000000000000000000 P 69844543993b4308a9c886a9fff341ca: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:46:10.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.329011 3915 master_runner.cc:186] Error getting master registration for 127.3.171.190:36307: OK, instance_id { permanent_uuid: "934dd201770e4f2b8cf9a0cc7ac7659a" instance_seqno: 1790567166618068 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
03:46:10.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.365368 3915 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:46:10.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:10.368414 4027 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 934dd201770e4f2b8cf9a0cc7ac7659a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:46:10.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:10.368935 4027 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:46:10.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.353549 3770 master_runner.cc:186] Error getting master registration for 127.3.171.189:33249: OK, instance_id { permanent_uuid: "a579ba8360cd48569bdc7938e46df544" instance_seqno: 1790567168388904 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
03:46:10.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.385752 3770 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:46:10.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.472195 4003 raft_consensus.cc:482] T 00000000000000000000000000000000 P 69844543993b4308a9c886a9fff341ca [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:46:10.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.472715 4003 raft_consensus.cc:504] T 00000000000000000000000000000000 P 69844543993b4308a9c886a9fff341ca [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "934dd201770e4f2b8cf9a0cc7ac7659a" member_type: VOTER last_known_addr { host: "127.3.171.190" port: 36307 } } peers { permanent_uuid: "a579ba8360cd48569bdc7938e46df544" member_type: VOTER last_known_addr { host: "127.3.171.189" port: 33249 } } peers { permanent_uuid: "69844543993b4308a9c886a9fff341ca" member_type: VOTER last_known_addr { host: "127.3.171.188" port: 38157 } }
03:46:10.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.475257 4003 leader_election.cc:288] T 00000000000000000000000000000000 P 69844543993b4308a9c886a9fff341ca [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 934dd201770e4f2b8cf9a0cc7ac7659a (127.3.171.190:36307), a579ba8360cd48569bdc7938e46df544 (127.3.171.189:33249)
03:46:10.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.476343 3813 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "69844543993b4308a9c886a9fff341ca" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "934dd201770e4f2b8cf9a0cc7ac7659a" is_pre_election: true
03:46:10.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.477213 3813 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 934dd201770e4f2b8cf9a0cc7ac7659a [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 69844543993b4308a9c886a9fff341ca in term 0.
03:46:10.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.478458 3943 leader_election.cc:302] T 00000000000000000000000000000000 P 69844543993b4308a9c886a9fff341ca [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 69844543993b4308a9c886a9fff341ca, 934dd201770e4f2b8cf9a0cc7ac7659a; no voters:
03:46:10.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.479354 4003 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 69844543993b4308a9c886a9fff341ca [term 0 FOLLOWER]: Leader pre-election won for term 1
03:46:10.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.479684 4003 raft_consensus.cc:482] T 00000000000000000000000000000000 P 69844543993b4308a9c886a9fff341ca [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
03:46:10.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.480002 4003 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 69844543993b4308a9c886a9fff341ca [term 0 FOLLOWER]: Advancing to term 1
03:46:10.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.486487 4003 raft_consensus.cc:504] T 00000000000000000000000000000000 P 69844543993b4308a9c886a9fff341ca [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "934dd201770e4f2b8cf9a0cc7ac7659a" member_type: VOTER last_known_addr { host: "127.3.171.190" port: 36307 } } peers { permanent_uuid: "a579ba8360cd48569bdc7938e46df544" member_type: VOTER last_known_addr { host: "127.3.171.189" port: 33249 } } peers { permanent_uuid: "69844543993b4308a9c886a9fff341ca" member_type: VOTER last_known_addr { host: "127.3.171.188" port: 38157 } }
03:46:10.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.489117 4003 leader_election.cc:288] T 00000000000000000000000000000000 P 69844543993b4308a9c886a9fff341ca [CANDIDATE]: Term 1 election: Requested vote from peers 934dd201770e4f2b8cf9a0cc7ac7659a (127.3.171.190:36307), a579ba8360cd48569bdc7938e46df544 (127.3.171.189:33249)
03:46:10.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.489065 3813 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "69844543993b4308a9c886a9fff341ca" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "934dd201770e4f2b8cf9a0cc7ac7659a"
03:46:10.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.489616 3813 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 934dd201770e4f2b8cf9a0cc7ac7659a [term 0 FOLLOWER]: Advancing to term 1
03:46:10.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.501358 3813 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 934dd201770e4f2b8cf9a0cc7ac7659a [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 69844543993b4308a9c886a9fff341ca in term 1.
03:46:10.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.502467 3943 leader_election.cc:302] T 00000000000000000000000000000000 P 69844543993b4308a9c886a9fff341ca [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 69844543993b4308a9c886a9fff341ca, 934dd201770e4f2b8cf9a0cc7ac7659a; no voters:
03:46:10.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.503319 4003 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 69844543993b4308a9c886a9fff341ca [term 1 FOLLOWER]: Leader election won for term 1
03:46:10.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.509735 4003 raft_consensus.cc:686] T 00000000000000000000000000000000 P 69844543993b4308a9c886a9fff341ca [term 1 LEADER]: Becoming Leader. State: Replica: 69844543993b4308a9c886a9fff341ca, State: Running, Role: LEADER
03:46:10.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.510588 4003 consensus_queue.cc:237] T 00000000000000000000000000000000 P 69844543993b4308a9c886a9fff341ca [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "934dd201770e4f2b8cf9a0cc7ac7659a" member_type: VOTER last_known_addr { host: "127.3.171.190" port: 36307 } } peers { permanent_uuid: "a579ba8360cd48569bdc7938e46df544" member_type: VOTER last_known_addr { host: "127.3.171.189" port: 33249 } } peers { permanent_uuid: "69844543993b4308a9c886a9fff341ca" member_type: VOTER last_known_addr { host: "127.3.171.188" port: 38157 } }
03:46:10.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.521167 4029 sys_catalog.cc:455] T 00000000000000000000000000000000 P 69844543993b4308a9c886a9fff341ca [sys.catalog]: SysCatalogTable state changed. Reason: New leader 69844543993b4308a9c886a9fff341ca. Latest consensus state: current_term: 1 leader_uuid: "69844543993b4308a9c886a9fff341ca" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "934dd201770e4f2b8cf9a0cc7ac7659a" member_type: VOTER last_known_addr { host: "127.3.171.190" port: 36307 } } peers { permanent_uuid: "a579ba8360cd48569bdc7938e46df544" member_type: VOTER last_known_addr { host: "127.3.171.189" port: 33249 } } peers { permanent_uuid: "69844543993b4308a9c886a9fff341ca" member_type: VOTER last_known_addr { host: "127.3.171.188" port: 38157 } } }
03:46:10.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.521956 4029 sys_catalog.cc:458] T 00000000000000000000000000000000 P 69844543993b4308a9c886a9fff341ca [sys.catalog]: This master's current role is: LEADER
03:46:10.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.525344 4031 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
03:46:10.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.538720 4031 catalog_manager.cc:1484] Initializing Kudu cluster ID...
03:46:10.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.556526 3813 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 934dd201770e4f2b8cf9a0cc7ac7659a [term 1 FOLLOWER]: Refusing update from remote peer 69844543993b4308a9c886a9fff341ca: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
03:46:10.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.559944 4029 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 69844543993b4308a9c886a9fff341ca [LEADER]: Connected to new peer: Peer: permanent_uuid: "934dd201770e4f2b8cf9a0cc7ac7659a" member_type: VOTER last_known_addr { host: "127.3.171.190" port: 36307 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:46:10.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.581619 4009 sys_catalog.cc:455] T 00000000000000000000000000000000 P 934dd201770e4f2b8cf9a0cc7ac7659a [sys.catalog]: SysCatalogTable state changed. Reason: New leader 69844543993b4308a9c886a9fff341ca. Latest consensus state: current_term: 1 leader_uuid: "69844543993b4308a9c886a9fff341ca" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "934dd201770e4f2b8cf9a0cc7ac7659a" member_type: VOTER last_known_addr { host: "127.3.171.190" port: 36307 } } peers { permanent_uuid: "a579ba8360cd48569bdc7938e46df544" member_type: VOTER last_known_addr { host: "127.3.171.189" port: 33249 } } peers { permanent_uuid: "69844543993b4308a9c886a9fff341ca" member_type: VOTER last_known_addr { host: "127.3.171.188" port: 38157 } } }
03:46:10.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.582346 4009 sys_catalog.cc:458] T 00000000000000000000000000000000 P 934dd201770e4f2b8cf9a0cc7ac7659a [sys.catalog]: This master's current role is: FOLLOWER
03:46:10.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.590365 4032 mvcc.cc:205] Tried to move back new op lower bound from 7334163130578706432 to 7334163130418462720. Current Snapshot: MvccSnapshot[applied={T|T < 7334163130578706432}]
03:46:10.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.594466 4003 sys_catalog.cc:455] T 00000000000000000000000000000000 P 69844543993b4308a9c886a9fff341ca [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "69844543993b4308a9c886a9fff341ca" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "934dd201770e4f2b8cf9a0cc7ac7659a" member_type: VOTER last_known_addr { host: "127.3.171.190" port: 36307 } } peers { permanent_uuid: "a579ba8360cd48569bdc7938e46df544" member_type: VOTER last_known_addr { host: "127.3.171.189" port: 33249 } } peers { permanent_uuid: "69844543993b4308a9c886a9fff341ca" member_type: VOTER last_known_addr { host: "127.3.171.188" port: 38157 } } }
03:46:10.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.595533 4003 sys_catalog.cc:458] T 00000000000000000000000000000000 P 69844543993b4308a9c886a9fff341ca [sys.catalog]: This master's current role is: LEADER
03:46:10.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.605163 4031 catalog_manager.cc:1347] Generated new cluster ID: 8de916d56af042a78255279ef3d1dc1a
03:46:10.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.605434 4031 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
03:46:10.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.618194 4009 sys_catalog.cc:455] T 00000000000000000000000000000000 P 934dd201770e4f2b8cf9a0cc7ac7659a [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "69844543993b4308a9c886a9fff341ca" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "934dd201770e4f2b8cf9a0cc7ac7659a" member_type: VOTER last_known_addr { host: "127.3.171.190" port: 36307 } } peers { permanent_uuid: "a579ba8360cd48569bdc7938e46df544" member_type: VOTER last_known_addr { host: "127.3.171.189" port: 33249 } } peers { permanent_uuid: "69844543993b4308a9c886a9fff341ca" member_type: VOTER last_known_addr { host: "127.3.171.188" port: 38157 } } }
03:46:10.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.618899 4009 sys_catalog.cc:458] T 00000000000000000000000000000000 P 934dd201770e4f2b8cf9a0cc7ac7659a [sys.catalog]: This master's current role is: FOLLOWER
03:46:10.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.645927 4031 catalog_manager.cc:1370] Generated new certificate authority record
03:46:10.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.653267 4031 catalog_manager.cc:1504] Loading token signing keys...
03:46:10.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.705729 4031 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 69844543993b4308a9c886a9fff341ca: Generated new TSK 0
03:46:10.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.710813 4031 catalog_manager.cc:1514] Initializing in-progress tserver states...
03:46:10.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.755645 3758 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskbYfj1w/build/tsan/bin/kudu
03:46:10.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbYfj1w/build/tsan/bin/kudu
03:46:10.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/ts-0/wal
03:46:10.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/ts-0/data
03:46:10.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:46:10.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:46:10.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:46:10.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:46:10.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:46:10.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:46:10.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/ts-0/logs
03:46:10.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/ts-0/data/info.pb
03:46:10.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:46:10.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:46:10.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:46:10.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:46:10.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:46:10.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:46:10.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:46:10.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:46:10.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:46:10.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:46:10.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.171.129:0
03:46:10.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.171.129
03:46:10.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.171.129
03:46:10.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:46:10.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.171.190:36307,127.3.171.189:33249,127.3.171.188:38157
03:46:10.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.171.148:45923
03:46:10.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:46:10.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:46:10.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
03:46:10.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.760634 3913 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.171.188" port: 38157 } attempt: 7
03:46:10.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.790879 3913 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a579ba8360cd48569bdc7938e46df544: Bootstrap starting.
03:46:10.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.804488 3913 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P a579ba8360cd48569bdc7938e46df544: Neither blocks nor log segments found. Creating new log.
03:46:10.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.806707 3913 log.cc:824] T 00000000000000000000000000000000 P a579ba8360cd48569bdc7938e46df544: Log is configured to *not* fsync() on all Append() calls
03:46:10.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.828995 3913 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a579ba8360cd48569bdc7938e46df544: No bootstrap required, opened a new log
03:46:10.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.858932 3913 raft_consensus.cc:348] T 00000000000000000000000000000000 P a579ba8360cd48569bdc7938e46df544 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "934dd201770e4f2b8cf9a0cc7ac7659a" member_type: VOTER last_known_addr { host: "127.3.171.190" port: 36307 } } peers { permanent_uuid: "a579ba8360cd48569bdc7938e46df544" member_type: VOTER last_known_addr { host: "127.3.171.189" port: 33249 } } peers { permanent_uuid: "69844543993b4308a9c886a9fff341ca" member_type: VOTER last_known_addr { host: "127.3.171.188" port: 38157 } }
03:46:10.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.860128 3913 raft_consensus.cc:374] T 00000000000000000000000000000000 P a579ba8360cd48569bdc7938e46df544 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:46:10.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.860610 3913 raft_consensus.cc:729] T 00000000000000000000000000000000 P a579ba8360cd48569bdc7938e46df544 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a579ba8360cd48569bdc7938e46df544, State: Initialized, Role: FOLLOWER
03:46:10.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.861720 3913 consensus_queue.cc:260] T 00000000000000000000000000000000 P a579ba8360cd48569bdc7938e46df544 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "934dd201770e4f2b8cf9a0cc7ac7659a" member_type: VOTER last_known_addr { host: "127.3.171.190" port: 36307 } } peers { permanent_uuid: "a579ba8360cd48569bdc7938e46df544" member_type: VOTER last_known_addr { host: "127.3.171.189" port: 33249 } } peers { permanent_uuid: "69844543993b4308a9c886a9fff341ca" member_type: VOTER last_known_addr { host: "127.3.171.188" port: 38157 } }
03:46:10.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.881096 3886 raft_consensus.cc:3037] T 00000000000000000000000000000000 P a579ba8360cd48569bdc7938e46df544 [term 0 FOLLOWER]: Advancing to term 1
03:46:10.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.885087 3887 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "69844543993b4308a9c886a9fff341ca" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a579ba8360cd48569bdc7938e46df544"
03:46:10.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.886551 3888 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "69844543993b4308a9c886a9fff341ca" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a579ba8360cd48569bdc7938e46df544" is_pre_election: true
03:46:10.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.892035 3886 raft_consensus.cc:1261] T 00000000000000000000000000000000 P a579ba8360cd48569bdc7938e46df544 [term 1 FOLLOWER]: Refusing update from remote peer 69844543993b4308a9c886a9fff341ca: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
03:46:10.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.893427 3913 sys_catalog.cc:564] T 00000000000000000000000000000000 P a579ba8360cd48569bdc7938e46df544 [sys.catalog]: configured and running, proceeding with master startup.
03:46:10.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.893669 4003 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 69844543993b4308a9c886a9fff341ca [LEADER]: Connected to new peer: Peer: permanent_uuid: "a579ba8360cd48569bdc7938e46df544" member_type: VOTER last_known_addr { host: "127.3.171.189" port: 33249 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:46:10.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.894111 4041 sys_catalog.cc:455] T 00000000000000000000000000000000 P a579ba8360cd48569bdc7938e46df544 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "934dd201770e4f2b8cf9a0cc7ac7659a" member_type: VOTER last_known_addr { host: "127.3.171.190" port: 36307 } } peers { permanent_uuid: "a579ba8360cd48569bdc7938e46df544" member_type: VOTER last_known_addr { host: "127.3.171.189" port: 33249 } } peers { permanent_uuid: "69844543993b4308a9c886a9fff341ca" member_type: VOTER last_known_addr { host: "127.3.171.188" port: 38157 } } }
03:46:10.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.898764 4041 sys_catalog.cc:458] T 00000000000000000000000000000000 P a579ba8360cd48569bdc7938e46df544 [sys.catalog]: This master's current role is: FOLLOWER
03:46:10.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.958976 4042 sys_catalog.cc:455] T 00000000000000000000000000000000 P a579ba8360cd48569bdc7938e46df544 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "69844543993b4308a9c886a9fff341ca" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "934dd201770e4f2b8cf9a0cc7ac7659a" member_type: VOTER last_known_addr { host: "127.3.171.190" port: 36307 } } peers { permanent_uuid: "a579ba8360cd48569bdc7938e46df544" member_type: VOTER last_known_addr { host: "127.3.171.189" port: 33249 } } peers { permanent_uuid: "69844543993b4308a9c886a9fff341ca" member_type: VOTER last_known_addr { host: "127.3.171.188" port: 38157 } } }
03:46:10.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.961207 4042 sys_catalog.cc:458] T 00000000000000000000000000000000 P a579ba8360cd48569bdc7938e46df544 [sys.catalog]: This master's current role is: FOLLOWER
03:46:10.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.960384 4041 sys_catalog.cc:455] T 00000000000000000000000000000000 P a579ba8360cd48569bdc7938e46df544 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 69844543993b4308a9c886a9fff341ca. Latest consensus state: current_term: 1 leader_uuid: "69844543993b4308a9c886a9fff341ca" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "934dd201770e4f2b8cf9a0cc7ac7659a" member_type: VOTER last_known_addr { host: "127.3.171.190" port: 36307 } } peers { permanent_uuid: "a579ba8360cd48569bdc7938e46df544" member_type: VOTER last_known_addr { host: "127.3.171.189" port: 33249 } } peers { permanent_uuid: "69844543993b4308a9c886a9fff341ca" member_type: VOTER last_known_addr { host: "127.3.171.188" port: 38157 } } }
03:46:10.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:10.961927 4041 sys_catalog.cc:458] T 00000000000000000000000000000000 P a579ba8360cd48569bdc7938e46df544 [sys.catalog]: This master's current role is: FOLLOWER
03:46:11.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:11.006414 4057 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
03:46:11.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:11.218897 4038 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:46:11.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:11.219861 4038 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:46:11.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:11.220788 4038 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:46:11.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:11.276098 4038 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:46:11.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:11.277509 4038 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.3.171.129
03:46:11.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:11.337441 4038 tablet_server_runner.cc:78] Tablet server non-default flags:
03:46:11.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:46:11.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.171.148:45923
03:46:11.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:46:11.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:46:11.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/ts-0/data
03:46:11.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/ts-0/wal
03:46:11.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:46:11.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:46:11.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.171.129:0
03:46:11.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:46:11.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:46:11.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:46:11.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/ts-0/data/info.pb
03:46:11.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.171.129
03:46:11.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:46:11.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.171.190:36307,127.3.171.189:33249,127.3.171.188:38157
03:46:11.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:46:11.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:46:11.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:46:11.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:46:11.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:46:11.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.171.129
03:46:11.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/ts-0/logs
03:46:11.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:46:11.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:46:11.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:46:11.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:46:11.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:46:11.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:46:11.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:46:11.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 02:50:13 UTC on ccycloud.quasar-purgqw.root.comops.site
03:46:11.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676527
03:46:11.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:46:11.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:11.339015 4038 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:46:11.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:11.341069 4038 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:46:11.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:11.365187 4063 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:46:11.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:11.375155 4027 catalog_manager.cc:1259] Loaded cluster ID: 8de916d56af042a78255279ef3d1dc1a
03:46:11.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:11.375737 4027 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 934dd201770e4f2b8cf9a0cc7ac7659a: loading cluster ID for follower catalog manager: success
03:46:11.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:11.384318 4027 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 934dd201770e4f2b8cf9a0cc7ac7659a: acquiring CA information for follower catalog manager: success
03:46:11.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:11.405092 4027 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 934dd201770e4f2b8cf9a0cc7ac7659a: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:46:11.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:11.423183 4068 sys_catalog.cc:455] T 00000000000000000000000000000000 P 69844543993b4308a9c886a9fff341ca [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "69844543993b4308a9c886a9fff341ca" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "934dd201770e4f2b8cf9a0cc7ac7659a" member_type: VOTER last_known_addr { host: "127.3.171.190" port: 36307 } } peers { permanent_uuid: "a579ba8360cd48569bdc7938e46df544" member_type: VOTER last_known_addr { host: "127.3.171.189" port: 33249 } } peers { permanent_uuid: "69844543993b4308a9c886a9fff341ca" member_type: VOTER last_known_addr { host: "127.3.171.188" port: 38157 } } }
03:46:11.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:11.436110 4068 sys_catalog.cc:458] T 00000000000000000000000000000000 P 69844543993b4308a9c886a9fff341ca [sys.catalog]: This master's current role is: LEADER
03:46:12.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:12.048540 4057 catalog_manager.cc:1259] Loaded cluster ID: 8de916d56af042a78255279ef3d1dc1a
03:46:12.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:12.052814 4057 catalog_manager.cc:1552] T 00000000000000000000000000000000 P a579ba8360cd48569bdc7938e46df544: loading cluster ID for follower catalog manager: success
03:46:12.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:12.092197 4057 catalog_manager.cc:1574] T 00000000000000000000000000000000 P a579ba8360cd48569bdc7938e46df544: acquiring CA information for follower catalog manager: success
03:46:12.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:12.126084 4057 catalog_manager.cc:1602] T 00000000000000000000000000000000 P a579ba8360cd48569bdc7938e46df544: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:46:13.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:12.757694 4062 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 4038
03:46:13.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:13.495841 4066 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:46:13.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:13.491650 4038 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 2.124s user 0.000s sys 0.001s
03:46:13.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:13.496282 4038 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 2.129s user 0.001s sys 0.001s
03:46:13.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:13.496528 4038 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:46:13.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:11.365600 4064 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:46:13.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:13.495841 4065 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2127 milliseconds
03:46:13.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:13.500959 4038 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:46:13.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:13.503752 4038 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:46:13.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:13.505174 4038 hybrid_clock.cc:648] HybridClock initialized: now 1790567173505131 us; error 53 us; skew 500 ppm
03:46:13.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:13.506003 4038 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:46:13.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:13.518447 4038 webserver.cc:460] Webserver started at http://127.3.171.129:45551/ using document root <none> and password file <none>
03:46:13.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:13.519641 4038 fs_manager.cc:359] Metadata directory not provided
03:46:13.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:13.519907 4038 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:46:13.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:13.520433 4038 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:46:13.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:13.526384 4038 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/ts-0/data/instance:
03:46:13.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d92ebd4679b44d62bc018a4087b11e10"
03:46:13.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 03:46:13 on dist-test-slave-n4rz"
03:46:13.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:13.527580 4038 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/ts-0/wal/instance:
03:46:13.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d92ebd4679b44d62bc018a4087b11e10"
03:46:13.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 03:46:13 on dist-test-slave-n4rz"
03:46:13.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:13.535157 4038 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
03:46:13.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:13.541078 4080 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:46:13.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:13.542189 4038 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
03:46:13.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:13.542505 4038 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/ts-0/data,/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/ts-0/wal
03:46:13.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d92ebd4679b44d62bc018a4087b11e10"
03:46:13.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 03:46:13 on dist-test-slave-n4rz"
03:46:13.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:13.542841 4038 fs_report.cc:383] FS layout report
03:46:13.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:46:13.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/ts-0/wal
03:46:13.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/ts-0/wal
03:46:13.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/ts-0/data/data
03:46:13.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:46:13.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:46:13.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:46:13.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:46:13.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:46:13.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:46:13.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:46:13.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:46:13.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:46:13.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:46:13.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:46:13.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:13.597234 4038 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:46:13.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:13.599153 4038 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:46:13.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:13.599740 4038 kserver.cc:163] Server-wide thread pool size limit: 3276
03:46:13.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:13.605654 4038 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
03:46:13.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:13.619478 4038 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:46:13.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:13.619809 4038 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.001s
03:46:13.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:13.620087 4038 ts_tablet_manager.cc:610] Registered 0 tablets
03:46:13.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:13.620299 4038 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
03:46:13.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:13.843029 4038 rpc_server.cc:307] RPC server started. Bound to: 127.3.171.129:41789
03:46:13.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:13.843169 4192 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.3.171.129:41789 every 8 connection(s)
03:46:13.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:13.846331 4038 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/ts-0/data/info.pb
03:46:13.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:13.856189 3758 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskbYfj1w/build/tsan/bin/kudu as pid 4038
03:46:13.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:13.856623 3758 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/ts-0/wal/instance
03:46:13.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:13.890687 3758 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskbYfj1w/build/tsan/bin/kudu
03:46:13.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbYfj1w/build/tsan/bin/kudu
03:46:13.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/ts-1/wal
03:46:13.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/ts-1/data
03:46:13.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:46:13.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:46:13.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:46:13.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:46:13.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:46:13.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:46:13.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/ts-1/logs
03:46:13.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/ts-1/data/info.pb
03:46:13.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:46:13.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:46:13.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:46:13.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:46:13.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:46:13.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:46:13.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:46:13.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:46:13.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:46:13.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:46:13.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.171.130:0
03:46:13.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.171.130
03:46:13.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.171.130
03:46:13.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:46:13.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.171.190:36307,127.3.171.189:33249,127.3.171.188:38157
03:46:13.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.171.148:45923
03:46:13.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:46:13.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:46:13.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
03:46:13.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:13.932869 4193 heartbeater.cc:347] Connected to a master server at 127.3.171.188:38157
03:46:13.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:13.933640 4193 heartbeater.cc:464] Registering TS with master...
03:46:13.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:13.934974 4193 heartbeater.cc:511] Master 127.3.171.188:38157 requested a full tablet report, sending...
03:46:13.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:13.938092 3957 ts_manager.cc:194] Registered new tserver with Master: d92ebd4679b44d62bc018a4087b11e10 (127.3.171.129:41789)
03:46:13.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:13.941690 4194 heartbeater.cc:347] Connected to a master server at 127.3.171.189:33249
03:46:13.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:13.941967 4194 heartbeater.cc:464] Registering TS with master...
03:46:13.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:13.941797 3957 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.3.171.129:37777
03:46:13.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:13.942692 4194 heartbeater.cc:511] Master 127.3.171.189:33249 requested a full tablet report, sending...
03:46:13.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:13.945537 3878 ts_manager.cc:194] Registered new tserver with Master: d92ebd4679b44d62bc018a4087b11e10 (127.3.171.129:41789)
03:46:13.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:13.945853 4197 heartbeater.cc:347] Connected to a master server at 127.3.171.190:36307
03:46:13.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:13.946177 4197 heartbeater.cc:464] Registering TS with master...
03:46:13.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:13.946877 4197 heartbeater.cc:511] Master 127.3.171.190:36307 requested a full tablet report, sending...
03:46:13.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:13.949471 3803 ts_manager.cc:194] Registered new tserver with Master: d92ebd4679b44d62bc018a4087b11e10 (127.3.171.129:41789)
03:46:14.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:14.302964 4203 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:46:14.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:14.303422 4203 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:46:14.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:14.303836 4203 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:46:14.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:14.334975 4203 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:46:14.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:14.335717 4203 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.3.171.130
03:46:14.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:14.373314 4203 tablet_server_runner.cc:78] Tablet server non-default flags:
03:46:14.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:46:14.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.171.148:45923
03:46:14.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:46:14.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:46:14.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/ts-1/data
03:46:14.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/ts-1/wal
03:46:14.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:46:14.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:46:14.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.171.130:0
03:46:14.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:46:14.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:46:14.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:46:14.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/ts-1/data/info.pb
03:46:14.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.171.130
03:46:14.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:46:14.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.171.190:36307,127.3.171.189:33249,127.3.171.188:38157
03:46:14.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:46:14.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:46:14.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:46:14.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:46:14.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:46:14.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.171.130
03:46:14.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/ts-1/logs
03:46:14.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:46:14.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:46:14.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:46:14.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:46:14.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:46:14.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:46:14.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:46:14.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 02:50:13 UTC on ccycloud.quasar-purgqw.root.comops.site
03:46:14.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676527
03:46:14.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:46:14.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:14.374531 4203 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:46:14.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:14.376240 4203 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:46:14.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:14.399473 4210 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:46:14.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:14.599946 3909 debug-util.cc:398] Leaking SignalData structure 0x7b0800084720 after lost signal to thread 3844
03:46:14.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:14.951854 4193 heartbeater.cc:503] Master 127.3.171.188:38157 was elected leader, sending a full tablet report...
03:46:16.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:14.399549 4212 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:46:16.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:14.400669 4209 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:46:16.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:16.060166 4203 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:46:16.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:16.060683 4211 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1662 milliseconds
03:46:16.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:16.075446 4203 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:46:16.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:16.080310 4203 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:46:16.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:16.081743 4203 hybrid_clock.cc:648] HybridClock initialized: now 1790567176081713 us; error 49 us; skew 500 ppm
03:46:16.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:16.082882 4203 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:46:16.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:16.097889 4203 webserver.cc:460] Webserver started at http://127.3.171.130:38923/ using document root <none> and password file <none>
03:46:16.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:16.099169 4203 fs_manager.cc:359] Metadata directory not provided
03:46:16.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:16.099443 4203 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:46:16.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:16.100013 4203 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:46:16.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:16.107389 4203 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/ts-1/data/instance:
03:46:16.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0147bf8005934db0b89cdd39b2ca545d"
03:46:16.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 03:46:16 on dist-test-slave-n4rz"
03:46:16.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:16.108946 4203 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/ts-1/wal/instance:
03:46:16.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0147bf8005934db0b89cdd39b2ca545d"
03:46:16.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 03:46:16 on dist-test-slave-n4rz"
03:46:16.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:16.118692 4203 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.004s sys 0.004s
03:46:16.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:16.125308 4222 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:46:16.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:16.126611 4203 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.000s sys 0.004s
03:46:16.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:16.126878 4203 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/ts-1/data,/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/ts-1/wal
03:46:16.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0147bf8005934db0b89cdd39b2ca545d"
03:46:16.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 03:46:16 on dist-test-slave-n4rz"
03:46:16.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:16.127224 4203 fs_report.cc:383] FS layout report
03:46:16.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:46:16.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/ts-1/wal
03:46:16.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/ts-1/wal
03:46:16.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/ts-1/data/data
03:46:16.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:46:16.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:46:16.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:46:16.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:46:16.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:46:16.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:46:16.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:46:16.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:46:16.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:46:16.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:46:16.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:46:16.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:16.180197 4203 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:46:16.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:16.181926 4203 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:46:16.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:16.182466 4203 kserver.cc:163] Server-wide thread pool size limit: 3276
03:46:16.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:16.185693 4203 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
03:46:16.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:16.191200 4203 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:46:16.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:16.191449 4203 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
03:46:16.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:16.191731 4203 ts_tablet_manager.cc:610] Registered 0 tablets
03:46:16.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:16.191921 4203 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
03:46:16.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:16.366590 4203 rpc_server.cc:307] RPC server started. Bound to: 127.3.171.130:43729
03:46:16.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:16.366678 4334 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.3.171.130:43729 every 8 connection(s)
03:46:16.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:16.369180 4203 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/ts-1/data/info.pb
03:46:16.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:16.376884 3758 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskbYfj1w/build/tsan/bin/kudu as pid 4203
03:46:16.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:16.377554 3758 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/ts-1/wal/instance
03:46:16.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:16.411768 3758 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskbYfj1w/build/tsan/bin/kudu
03:46:16.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbYfj1w/build/tsan/bin/kudu
03:46:16.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/ts-2/wal
03:46:16.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/ts-2/data
03:46:16.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:46:16.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:46:16.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:46:16.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:46:16.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:46:16.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:46:16.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/ts-2/logs
03:46:16.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/ts-2/data/info.pb
03:46:16.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:46:16.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:46:16.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:46:16.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:46:16.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:46:16.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:46:16.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:46:16.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:46:16.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:46:16.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:46:16.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.171.131:0
03:46:16.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.171.131
03:46:16.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.171.131
03:46:16.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:46:16.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.171.190:36307,127.3.171.189:33249,127.3.171.188:38157
03:46:16.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.171.148:45923
03:46:16.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:46:16.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:46:16.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
03:46:16.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:16.448781 4336 heartbeater.cc:347] Connected to a master server at 127.3.171.189:33249
03:46:16.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:16.449515 4336 heartbeater.cc:464] Registering TS with master...
03:46:16.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:16.450697 4336 heartbeater.cc:511] Master 127.3.171.189:33249 requested a full tablet report, sending...
03:46:16.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:16.453147 4335 heartbeater.cc:347] Connected to a master server at 127.3.171.188:38157
03:46:16.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:16.453449 4335 heartbeater.cc:464] Registering TS with master...
03:46:16.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:16.454186 4335 heartbeater.cc:511] Master 127.3.171.188:38157 requested a full tablet report, sending...
03:46:16.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:16.454303 3878 ts_manager.cc:194] Registered new tserver with Master: 0147bf8005934db0b89cdd39b2ca545d (127.3.171.130:43729)
03:46:16.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:16.456117 3957 ts_manager.cc:194] Registered new tserver with Master: 0147bf8005934db0b89cdd39b2ca545d (127.3.171.130:43729)
03:46:16.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:16.457931 3957 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.3.171.130:45401
03:46:16.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:16.461289 4337 heartbeater.cc:347] Connected to a master server at 127.3.171.190:36307
03:46:16.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:16.461575 4337 heartbeater.cc:464] Registering TS with master...
03:46:16.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:16.462376 4337 heartbeater.cc:511] Master 127.3.171.190:36307 requested a full tablet report, sending...
03:46:16.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:16.465000 3803 ts_manager.cc:194] Registered new tserver with Master: 0147bf8005934db0b89cdd39b2ca545d (127.3.171.130:43729)
03:46:16.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:16.749344 4345 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:46:16.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:16.749897 4345 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:46:16.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:16.750517 4345 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:46:16.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:16.781581 4345 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:46:16.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:16.782461 4345 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.3.171.131
03:46:16.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:16.816661 4345 tablet_server_runner.cc:78] Tablet server non-default flags:
03:46:16.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:46:16.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.171.148:45923
03:46:16.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:46:16.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:46:16.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/ts-2/data
03:46:16.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/ts-2/wal
03:46:16.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:46:16.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:46:16.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.171.131:0
03:46:16.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:46:16.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:46:16.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:46:16.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/ts-2/data/info.pb
03:46:16.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.171.131
03:46:16.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:46:16.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.171.190:36307,127.3.171.189:33249,127.3.171.188:38157
03:46:16.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:46:16.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:46:16.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:46:16.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:46:16.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:46:16.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.171.131
03:46:16.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/ts-2/logs
03:46:16.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:46:16.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:46:16.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:46:16.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:46:16.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:46:16.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:46:16.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:46:16.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 02:50:13 UTC on ccycloud.quasar-purgqw.root.comops.site
03:46:16.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676527
03:46:16.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:46:16.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:16.818182 4345 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:46:16.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:16.820011 4345 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:46:16.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:16.841379 4351 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:46:17.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:17.113577 4189 debug-util.cc:398] Leaking SignalData structure 0x7b08000bb3e0 after lost signal to thread 4058
03:46:17.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:17.115852 4189 debug-util.cc:398] Leaking SignalData structure 0x7b08000afcc0 after lost signal to thread 4192
03:46:17.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:17.463151 4335 heartbeater.cc:503] Master 127.3.171.188:38157 was elected leader, sending a full tablet report...
03:46:18.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:16.842053 4352 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:46:18.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:16.841785 4354 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:46:18.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:18.336823 4353 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1493 milliseconds
03:46:18.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:18.336977 4345 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:46:18.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:18.338201 4345 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:46:18.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:18.345542 4345 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:46:18.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:18.346961 4345 hybrid_clock.cc:648] HybridClock initialized: now 1790567178346904 us; error 74 us; skew 500 ppm
03:46:18.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:18.347689 4345 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:46:18.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:18.354736 4345 webserver.cc:460] Webserver started at http://127.3.171.131:46169/ using document root <none> and password file <none>
03:46:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:18.355559 4345 fs_manager.cc:359] Metadata directory not provided
03:46:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:18.355742 4345 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:46:18.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:18.356138 4345 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:46:18.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:18.361322 4345 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/ts-2/data/instance:
03:46:18.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "56bfbe2e533e4a9fbc11bc02c4862bbe"
03:46:18.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 03:46:18 on dist-test-slave-n4rz"
03:46:18.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:18.364017 4345 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/ts-2/wal/instance:
03:46:18.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "56bfbe2e533e4a9fbc11bc02c4862bbe"
03:46:18.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 03:46:18 on dist-test-slave-n4rz"
03:46:18.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:18.373512 4345 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.008s sys 0.000s
03:46:18.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:18.381319 4362 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:46:18.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:18.382418 4345 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.001s sys 0.003s
03:46:18.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:18.382761 4345 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/ts-2/data,/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/ts-2/wal
03:46:18.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "56bfbe2e533e4a9fbc11bc02c4862bbe"
03:46:18.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 03:46:18 on dist-test-slave-n4rz"
03:46:18.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:18.383107 4345 fs_report.cc:383] FS layout report
03:46:18.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:46:18.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/ts-2/wal
03:46:18.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/ts-2/wal
03:46:18.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/ts-2/data/data
03:46:18.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:46:18.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:46:18.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:46:18.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:46:18.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:46:18.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:46:18.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:46:18.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:46:18.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:46:18.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:46:18.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:46:18.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:18.418632 4345 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:46:18.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:18.420557 4345 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:46:18.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:18.421132 4345 kserver.cc:163] Server-wide thread pool size limit: 3276
03:46:18.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:18.424397 4345 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
03:46:18.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:18.430272 4345 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:46:18.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:18.430562 4345 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
03:46:18.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:18.430898 4345 ts_tablet_manager.cc:610] Registered 0 tablets
03:46:18.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:18.431128 4345 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
03:46:18.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:18.592872 4345 rpc_server.cc:307] RPC server started. Bound to: 127.3.171.131:43405
03:46:18.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:18.592978 4474 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.3.171.131:43405 every 8 connection(s)
03:46:18.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:18.596665 4345 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/ts-2/data/info.pb
03:46:18.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:18.604476 3758 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskbYfj1w/build/tsan/bin/kudu as pid 4345
03:46:18.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:18.604974 3758 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster2088410077908773260/ts-2/wal/instance
03:46:18.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:18.660842 4476 heartbeater.cc:347] Connected to a master server at 127.3.171.189:33249
03:46:18.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:18.661410 4476 heartbeater.cc:464] Registering TS with master...
03:46:18.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:18.661983 4477 heartbeater.cc:347] Connected to a master server at 127.3.171.190:36307
03:46:18.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:18.662302 4477 heartbeater.cc:464] Registering TS with master...
03:46:18.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:18.662644 4476 heartbeater.cc:511] Master 127.3.171.189:33249 requested a full tablet report, sending...
03:46:18.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:18.663291 4477 heartbeater.cc:511] Master 127.3.171.190:36307 requested a full tablet report, sending...
03:46:18.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:18.665747 4475 heartbeater.cc:347] Connected to a master server at 127.3.171.188:38157
03:46:18.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:18.666069 4475 heartbeater.cc:464] Registering TS with master...
03:46:18.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:18.666199 3803 ts_manager.cc:194] Registered new tserver with Master: 56bfbe2e533e4a9fbc11bc02c4862bbe (127.3.171.131:43405)
03:46:18.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:18.666173 3878 ts_manager.cc:194] Registered new tserver with Master: 56bfbe2e533e4a9fbc11bc02c4862bbe (127.3.171.131:43405)
03:46:18.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:18.666836 4475 heartbeater.cc:511] Master 127.3.171.188:38157 requested a full tablet report, sending...
03:46:18.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:18.668771 3957 ts_manager.cc:194] Registered new tserver with Master: 56bfbe2e533e4a9fbc11bc02c4862bbe (127.3.171.131:43405)
03:46:18.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:18.670989 3957 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.3.171.131:34367
03:46:18.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:18.685220 3758 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
03:46:18.700 [DEBUG - main] (MiniKuduCluster.java:195) Response:
03:46:18.701 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
03:46:18.712 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.3.171.190"
port: 36307
}
bound_http_address {
host: "127.3.171.190"
port: 44493
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.3.171.189"
port: 33249
}
bound_http_address {
host: "127.3.171.189"
port: 43357
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.3.171.188"
port: 38157
}
bound_http_address {
host: "127.3.171.188"
port: 43679
}
}
}
03:46:18.717 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
03:46:18.721 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.3.171.129"
port: 41789
}
bound_http_address {
host: "127.3.171.129"
port: 45551
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.3.171.130"
port: 43729
}
bound_http_address {
host: "127.3.171.130"
port: 38923
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.3.171.131"
port: 43405
}
bound_http_address {
host: "127.3.171.131"
port: 46169
}
}
}
03:46:18.721 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
03:46:18.896 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
03:46:19.047 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskbYfj1w/test-tmp
03:46:19.048 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
03:46:19.049 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskbYfj1w/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
03:46:19.051 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
03:46:19.410 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:46:19.410 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
03:46:19.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:19.413537 4487 mini_oidc.cc:152] Starting JWKS server
03:46:19.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:19.416983 4487 webserver.cc:460] Webserver started at http://127.0.0.1:45935/ using document root <none> and password file <none>
03:46:19.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:19.417368 4487 mini_oidc.cc:189] Starting OIDC Discovery server
03:46:19.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:19.419270 4487 webserver.cc:460] Webserver started at http://127.0.0.1:43621/ using document root <none> and password file <none>
03:46:19.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
03:46:19.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/krb5kdc/principal' for realm 'KRBTEST.COM',
03:46:19.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
03:46:19.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 03:46:19 dist-test-slave-n4rz krb5kdc[4499](info): setting up network...
03:46:19.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
03:46:19.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 03:46:19 dist-test-slave-n4rz krb5kdc[4499](info): set up 2 sockets
03:46:19.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 03:46:19 dist-test-slave-n4rz krb5kdc[4499](info): commencing operation
03:46:19.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
03:46:19.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:19.674787 4475 heartbeater.cc:503] Master 127.3.171.188:38157 was elected leader, sending a full tablet report...
03:46:20.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:20.527163 4487 mini_kdc.cc:121] Time spent starting KDC: real 1.084s user 0.001s sys 0.019s
03:46:20.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
03:46:20.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
03:46:20.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
03:46:20.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
03:46:20.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
03:46:20.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
03:46:20.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
03:46:20.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
03:46:20.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
03:46:20.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
03:46:20.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/krb5kdc/test-user.keytab.
03:46:20.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/krb5kdc/test-user.keytab.
03:46:20.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 03:46:20 dist-test-slave-n4rz krb5kdc[4499](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790567180, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:46:20.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
03:46:20.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-28T03:46:20Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
03:46:20.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-28T03:46:20Z Disabled control of system clock
03:46:20.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.4.97.254@KRBTEST.COM; defaulting to no policy
03:46:20.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:46:20.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.4.97.254@KRBTEST.COM" created.
03:46:20.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:46:20.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.4.97.254 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/krb5kdc/kudu_127.4.97.254.keytab.
03:46:20.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.4.97.254 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/krb5kdc/kudu_127.4.97.254.keytab.
03:46:20.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:20.717708 4487 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskbYfj1w/build/tsan/bin/kudu
03:46:20.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbYfj1w/build/tsan/bin/kudu
03:46:20.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-0/wal
03:46:20.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-0/data
03:46:20.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:46:20.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:46:20.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:46:20.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:46:20.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:46:20.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:46:20.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-0/logs
03:46:20.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-0/data/info.pb
03:46:20.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:46:20.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:46:20.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:46:20.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:46:20.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:46:20.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:46:20.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:46:20.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:46:20.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:46:20.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:46:20.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:46:20.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:46:20.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.97.254:35997
03:46:20.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.97.254
03:46:20.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:46:20.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/krb5kdc/kudu_127.4.97.254.keytab
03:46:20.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.97.254
03:46:20.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:46:20.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:46:20.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:46:20.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.97.212:40151
03:46:20.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:46:20.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:46:20.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:46:20.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:46:20.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.97.254:35997,127.4.97.253:46769,127.4.97.252:45135
03:46:20.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:46:20.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45935/jwks.json
03:46:20.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:46:21.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:21.028256 4512 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:46:21.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:21.028899 4512 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:46:21.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:21.029567 4512 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:46:21.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:21.059917 4512 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:46:21.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:21.060128 4512 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:46:21.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:21.060530 4512 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:46:21.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:21.060837 4512 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:46:21.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:21.061163 4512 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:46:21.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:21.061514 4512 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:46:21.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:21.061776 4512 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45935/jwks.json
03:46:21.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:21.098510 4512 master_runner.cc:386] Master server non-default flags:
03:46:21.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:46:21.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.97.212:40151
03:46:21.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:46:21.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:46:21.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-0/data
03:46:21.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-0/wal
03:46:21.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:46:21.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:46:21.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.97.254:35997,127.4.97.253:46769,127.4.97.252:45135
03:46:21.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/krb5kdc/kudu_127.4.97.254.keytab
03:46:21.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.97.254
03:46:21.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:46:21.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:46:21.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:46:21.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.97.254:35997
03:46:21.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:46:21.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:46:21.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:46:21.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45935/jwks.json
03:46:21.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:46:21.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:46:21.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:46:21.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:46:21.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:46:21.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-0/data/info.pb
03:46:21.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.97.254
03:46:21.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:46:21.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:46:21.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:46:21.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:46:21.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:46:21.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:46:21.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-0/logs
03:46:21.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:46:21.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:46:21.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:46:21.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:46:21.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:46:21.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:46:21.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:46:21.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 02:50:13 UTC on ccycloud.quasar-purgqw.root.comops.site
03:46:21.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676527
03:46:21.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:46:21.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:21.099983 4512 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:46:21.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:21.101325 4512 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:46:21.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:21.119009 4519 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:46:22.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:21.119191 4521 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:46:22.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:21.119028 4518 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:46:22.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:22.414994 4520 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1294 milliseconds
03:46:22.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:22.415107 4512 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:46:22.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:22.416213 4512 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:46:22.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:22.418884 4512 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:46:22.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:22.420240 4512 hybrid_clock.cc:648] HybridClock initialized: now 1790567182420217 us; error 33 us; skew 500 ppm
03:46:22.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:22.421144 4512 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:46:22.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 03:46:22 dist-test-slave-n4rz krb5kdc[4499](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790567182, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.97.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:46:22.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:22.426901 4512 init.cc:377] Logged in from keytab as kudu/127.4.97.254@KRBTEST.COM (short username kudu)
03:46:22.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:22.432113 4512 webserver.cc:460] Webserver started at http://127.4.97.254:41607/ using document root <none> and password file <none>
03:46:22.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:22.433101 4512 fs_manager.cc:359] Metadata directory not provided
03:46:22.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:22.433331 4512 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:46:22.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:22.433835 4512 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:46:22.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:22.438592 4512 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-0/data/instance:
03:46:22.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1639beaa478b4f928ca7167f7af13082"
03:46:22.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 03:46:22 on dist-test-slave-n4rz"
03:46:22.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:22.439755 4512 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-0/wal/instance:
03:46:22.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1639beaa478b4f928ca7167f7af13082"
03:46:22.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 03:46:22 on dist-test-slave-n4rz"
03:46:22.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:22.446913 4512 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.008s sys 0.001s
03:46:22.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:22.452144 4530 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:46:22.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:22.453220 4512 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
03:46:22.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:22.453575 4512 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-0/data,/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-0/wal
03:46:22.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1639beaa478b4f928ca7167f7af13082"
03:46:22.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 03:46:22 on dist-test-slave-n4rz"
03:46:22.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:22.453926 4512 fs_report.cc:383] FS layout report
03:46:22.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:46:22.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-0/wal
03:46:22.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-0/wal
03:46:22.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-0/data/data
03:46:22.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:46:22.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:46:22.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:46:22.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:46:22.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:46:22.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:46:22.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:46:22.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:46:22.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:46:22.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:46:22.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:46:22.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:22.484190 4512 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:46:22.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:22.489845 4512 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:46:22.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:22.490355 4512 kserver.cc:163] Server-wide thread pool size limit: 3276
03:46:22.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:22.563184 4512 rpc_server.cc:307] RPC server started. Bound to: 127.4.97.254:35997
03:46:22.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:22.563308 4582 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.97.254:35997 every 8 connection(s)
03:46:22.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:22.565692 4512 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-0/data/info.pb
03:46:22.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:22.568017 4487 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskbYfj1w/build/tsan/bin/kudu as pid 4512
03:46:22.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:22.568516 4487 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-0/wal/instance
03:46:22.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:22.571501 4583 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:46:22.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:22.583158 4583 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.97.254" port: 35997 } has no permanent_uuid. Determining permanent_uuid...
03:46:22.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.4.97.253@KRBTEST.COM; defaulting to no policy
03:46:22.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:46:22.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.4.97.253@KRBTEST.COM" created.
03:46:22.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 03:46:22 dist-test-slave-n4rz krb5kdc[4499](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790567182, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.97.254@KRBTEST.COM for kudu/127.4.97.254@KRBTEST.COM
03:46:22.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:46:22.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.4.97.253 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/krb5kdc/kudu_127.4.97.253.keytab.
03:46:22.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.4.97.253 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/krb5kdc/kudu_127.4.97.253.keytab.
03:46:22.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:22.621244 4487 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskbYfj1w/build/tsan/bin/kudu
03:46:22.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbYfj1w/build/tsan/bin/kudu
03:46:22.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-1/wal
03:46:22.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-1/data
03:46:22.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:46:22.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:46:22.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:46:22.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:46:22.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:46:22.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:46:22.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-1/logs
03:46:22.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-1/data/info.pb
03:46:22.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:46:22.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:46:22.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:46:22.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:46:22.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:46:22.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:46:22.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:46:22.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:46:22.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:46:22.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:46:22.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:46:22.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:46:22.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.97.253:46769
03:46:22.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.97.253
03:46:22.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:46:22.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/krb5kdc/kudu_127.4.97.253.keytab
03:46:22.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.97.253
03:46:22.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:46:22.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:46:22.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:46:22.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.97.212:40151
03:46:22.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:46:22.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:46:22.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:46:22.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:46:22.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.97.254:35997,127.4.97.253:46769,127.4.97.252:45135
03:46:22.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:46:22.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45935/jwks.json
03:46:22.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:46:22.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:22.628574 4583 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.97.253" port: 46769 } has no permanent_uuid. Determining permanent_uuid...
03:46:22.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:22.630837 4533 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.97.253:46769: connect: Connection refused (error 111)
03:46:22.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:22.634752 4583 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.97.253:46769: Network error: Client connection negotiation failed: client connection to 127.4.97.253:46769: connect: Connection refused (error 111)
03:46:22.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:22.684201 4583 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.97.253" port: 46769 } attempt: 1
03:46:22.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:22.687609 4583 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.97.253:46769: Network error: Client connection negotiation failed: client connection to 127.4.97.253:46769: connect: Connection refused (error 111)
03:46:22.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:22.756016 4583 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.97.253" port: 46769 } attempt: 2
03:46:22.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:22.759618 4583 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.97.253:46769: Network error: Client connection negotiation failed: client connection to 127.4.97.253:46769: connect: Connection refused (error 111)
03:46:22.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:22.851032 4583 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.97.253" port: 46769 } attempt: 3
03:46:22.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:22.854641 4583 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.97.253:46769: Network error: Client connection negotiation failed: client connection to 127.4.97.253:46769: connect: Connection refused (error 111)
03:46:22.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:22.935981 4589 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:46:22.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:22.936522 4589 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:46:22.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:22.937100 4589 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:46:22.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:22.968125 4589 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:46:22.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:22.968407 4589 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:46:22.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:22.968688 4589 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:46:22.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:22.968957 4589 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:46:22.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:22.969202 4589 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:46:22.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:22.969394 4589 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:46:22.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:22.969530 4589 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45935/jwks.json
03:46:22.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:22.998039 4583 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.97.253" port: 46769 } attempt: 4
03:46:23.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:23.002481 4583 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.97.253:46769: Network error: Client connection negotiation failed: client connection to 127.4.97.253:46769: connect: Connection refused (error 111)
03:46:23.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:23.009474 4589 master_runner.cc:386] Master server non-default flags:
03:46:23.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:46:23.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.97.212:40151
03:46:23.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:46:23.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:46:23.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-1/data
03:46:23.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-1/wal
03:46:23.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:46:23.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:46:23.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.97.254:35997,127.4.97.253:46769,127.4.97.252:45135
03:46:23.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/krb5kdc/kudu_127.4.97.253.keytab
03:46:23.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.97.253
03:46:23.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:46:23.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:46:23.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:46:23.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.97.253:46769
03:46:23.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:46:23.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:46:23.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:46:23.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45935/jwks.json
03:46:23.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:46:23.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:46:23.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:46:23.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:46:23.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:46:23.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-1/data/info.pb
03:46:23.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.97.253
03:46:23.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:46:23.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:46:23.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:46:23.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:46:23.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:46:23.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:46:23.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-1/logs
03:46:23.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:46:23.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:46:23.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:46:23.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:46:23.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:46:23.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:46:23.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:46:23.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 02:50:13 UTC on ccycloud.quasar-purgqw.root.comops.site
03:46:23.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676527
03:46:23.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:46:23.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:23.010531 4589 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:46:23.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:23.011581 4589 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:46:23.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:23.028605 4597 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:46:23.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:23.301868 4583 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.97.253" port: 46769 } attempt: 5
03:46:23.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:23.306532 4583 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.97.253:46769: Network error: Client connection negotiation failed: client connection to 127.4.97.253:46769: connect: Connection refused (error 111)
03:46:23.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:23.853966 4583 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.97.253" port: 46769 } attempt: 6
03:46:23.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:23.861119 4583 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.97.253:46769: Network error: Client connection negotiation failed: client connection to 127.4.97.253:46769: connect: Connection refused (error 111)
03:46:24.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:23.029129 4599 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:46:24.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:23.030815 4589 server_base.cc:1034] running on GCE node
03:46:24.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:23.028812 4596 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:46:24.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:24.246548 4589 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:46:24.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:24.249082 4589 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:46:24.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:24.250429 4589 hybrid_clock.cc:648] HybridClock initialized: now 1790567184250412 us; error 55 us; skew 500 ppm
03:46:24.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:24.251176 4589 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:46:24.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 03:46:24 dist-test-slave-n4rz krb5kdc[4499](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790567184, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.97.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:46:24.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:24.263957 4589 init.cc:377] Logged in from keytab as kudu/127.4.97.253@KRBTEST.COM (short username kudu)
03:46:24.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:24.269148 4589 webserver.cc:460] Webserver started at http://127.4.97.253:41571/ using document root <none> and password file <none>
03:46:24.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:24.269992 4589 fs_manager.cc:359] Metadata directory not provided
03:46:24.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:24.270174 4589 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:46:24.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:24.270529 4589 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:46:24.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:24.275964 4589 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-1/data/instance:
03:46:24.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "53a3a7a4248242bbbb5a044216cf0ae8"
03:46:24.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 03:46:24 on dist-test-slave-n4rz"
03:46:24.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:24.277046 4589 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-1/wal/instance:
03:46:24.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "53a3a7a4248242bbbb5a044216cf0ae8"
03:46:24.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 03:46:24 on dist-test-slave-n4rz"
03:46:24.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:24.284101 4589 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.005s sys 0.000s
03:46:24.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:24.291546 4610 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:46:24.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:24.292589 4589 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.001s
03:46:24.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:24.292938 4589 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-1/data,/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-1/wal
03:46:24.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "53a3a7a4248242bbbb5a044216cf0ae8"
03:46:24.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 03:46:24 on dist-test-slave-n4rz"
03:46:24.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:24.293326 4589 fs_report.cc:383] FS layout report
03:46:24.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:46:24.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-1/wal
03:46:24.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-1/wal
03:46:24.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-1/data/data
03:46:24.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:46:24.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:46:24.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:46:24.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:46:24.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:46:24.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:46:24.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:46:24.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:46:24.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:46:24.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:46:24.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:46:24.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:24.332428 4589 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:46:24.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:24.337918 4589 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:46:24.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:24.338330 4589 kserver.cc:163] Server-wide thread pool size limit: 3276
03:46:24.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:24.413754 4589 rpc_server.cc:307] RPC server started. Bound to: 127.4.97.253:46769
03:46:24.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:24.413836 4661 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.97.253:46769 every 8 connection(s)
03:46:24.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:24.416554 4589 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-1/data/info.pb
03:46:24.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:24.421768 4662 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:46:24.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:24.422372 4487 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskbYfj1w/build/tsan/bin/kudu as pid 4589
03:46:24.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:24.422699 4487 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-1/wal/instance
03:46:24.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:24.436357 4662 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.97.254" port: 35997 } has no permanent_uuid. Determining permanent_uuid...
03:46:24.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.4.97.252@KRBTEST.COM; defaulting to no policy
03:46:24.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:46:24.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.4.97.252@KRBTEST.COM" created.
03:46:24.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 03:46:24 dist-test-slave-n4rz krb5kdc[4499](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790567184, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.97.253@KRBTEST.COM for kudu/127.4.97.254@KRBTEST.COM
03:46:24.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:46:24.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.4.97.252 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/krb5kdc/kudu_127.4.97.252.keytab.
03:46:24.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.4.97.252 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/krb5kdc/kudu_127.4.97.252.keytab.
03:46:24.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:24.469404 4487 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskbYfj1w/build/tsan/bin/kudu
03:46:24.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbYfj1w/build/tsan/bin/kudu
03:46:24.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-2/wal
03:46:24.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-2/data
03:46:24.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:46:24.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:46:24.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:46:24.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:46:24.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:46:24.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:46:24.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-2/logs
03:46:24.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-2/data/info.pb
03:46:24.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:46:24.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:46:24.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:46:24.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:46:24.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:46:24.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:46:24.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:46:24.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:46:24.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:46:24.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:46:24.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:46:24.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:46:24.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.97.252:45135
03:46:24.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.97.252
03:46:24.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:46:24.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/krb5kdc/kudu_127.4.97.252.keytab
03:46:24.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.97.252
03:46:24.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:46:24.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:46:24.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:46:24.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.97.212:40151
03:46:24.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:46:24.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:46:24.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:46:24.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:46:24.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.97.254:35997,127.4.97.253:46769,127.4.97.252:45135
03:46:24.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:46:24.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45935/jwks.json
03:46:24.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:46:24.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:24.473174 4662 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.97.253" port: 46769 } has no permanent_uuid. Determining permanent_uuid...
03:46:24.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 03:46:24 dist-test-slave-n4rz krb5kdc[4499](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790567184, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.97.253@KRBTEST.COM for kudu/127.4.97.253@KRBTEST.COM
03:46:24.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:24.499433 4662 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.97.252" port: 45135 } has no permanent_uuid. Determining permanent_uuid...
03:46:24.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:24.501912 4614 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.97.252:45135: connect: Connection refused (error 111)
03:46:24.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:24.506582 4662 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.97.252:45135: Network error: Client connection negotiation failed: client connection to 127.4.97.252:45135: connect: Connection refused (error 111)
03:46:24.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:24.556066 4662 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.97.252" port: 45135 } attempt: 1
03:46:24.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:24.560289 4662 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.97.252:45135: Network error: Client connection negotiation failed: client connection to 127.4.97.252:45135: connect: Connection refused (error 111)
03:46:24.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:24.628726 4662 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.97.252" port: 45135 } attempt: 2
03:46:24.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:24.632606 4662 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.97.252:45135: Network error: Client connection negotiation failed: client connection to 127.4.97.252:45135: connect: Connection refused (error 111)
03:46:24.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:24.724047 4662 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.97.252" port: 45135 } attempt: 3
03:46:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:24.727607 4662 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.97.252:45135: Network error: Client connection negotiation failed: client connection to 127.4.97.252:45135: connect: Connection refused (error 111)
03:46:24.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:24.797279 4668 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:46:24.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:24.797845 4668 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:46:24.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:24.798414 4668 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:46:24.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:24.830259 4668 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:46:24.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:24.830470 4668 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:46:24.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:24.830725 4668 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:46:24.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:24.830915 4668 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:46:24.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:24.831164 4668 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:46:24.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:24.831352 4668 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:46:24.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:24.831488 4668 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45935/jwks.json
03:46:24.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:24.866765 4668 master_runner.cc:386] Master server non-default flags:
03:46:24.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:46:24.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.97.212:40151
03:46:24.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:46:24.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:46:24.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-2/data
03:46:24.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-2/wal
03:46:24.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:46:24.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:46:24.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.97.254:35997,127.4.97.253:46769,127.4.97.252:45135
03:46:24.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/krb5kdc/kudu_127.4.97.252.keytab
03:46:24.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.97.252
03:46:24.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:46:24.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:46:24.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:46:24.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.97.252:45135
03:46:24.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:46:24.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:46:24.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:46:24.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45935/jwks.json
03:46:24.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:46:24.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:46:24.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:46:24.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:46:24.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:46:24.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-2/data/info.pb
03:46:24.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.97.252
03:46:24.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:46:24.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:46:24.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:46:24.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:46:24.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:46:24.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:46:24.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-2/logs
03:46:24.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:46:24.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:46:24.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:46:24.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:46:24.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:46:24.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:46:24.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:46:24.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 02:50:13 UTC on ccycloud.quasar-purgqw.root.comops.site
03:46:24.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676527
03:46:24.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:46:24.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:24.867990 4668 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:46:24.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:24.868973 4668 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:46:24.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:24.871007 4662 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.97.252" port: 45135 } attempt: 4
03:46:24.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:24.874553 4662 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.97.252:45135: Network error: Client connection negotiation failed: client connection to 127.4.97.252:45135: connect: Connection refused (error 111)
03:46:24.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:24.888015 4677 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:46:24.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:24.921531 4583 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.97.253" port: 46769 } attempt: 7
03:46:24.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 03:46:24 dist-test-slave-n4rz krb5kdc[4499](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790567182, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.97.254@KRBTEST.COM for kudu/127.4.97.253@KRBTEST.COM
03:46:24.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:24.950816 4583 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.97.252" port: 45135 } has no permanent_uuid. Determining permanent_uuid...
03:46:24.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:24.957690 4583 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.97.252:45135: Network error: Client connection negotiation failed: client connection to 127.4.97.252:45135: connect: Connection refused (error 111)
03:46:25.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:25.016104 4583 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.97.252" port: 45135 } attempt: 1
03:46:25.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:25.019587 4583 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.97.252:45135: Network error: Client connection negotiation failed: client connection to 127.4.97.252:45135: connect: Connection refused (error 111)
03:46:25.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:25.100975 4583 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.97.252" port: 45135 } attempt: 2
03:46:25.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:25.105559 4583 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.97.252:45135: Network error: Client connection negotiation failed: client connection to 127.4.97.252:45135: connect: Connection refused (error 111)
03:46:25.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:25.173964 4662 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.97.252" port: 45135 } attempt: 5
03:46:25.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:25.178825 4662 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.97.252:45135: Network error: Client connection negotiation failed: client connection to 127.4.97.252:45135: connect: Connection refused (error 111)
03:46:25.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:25.190917 4583 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.97.252" port: 45135 } attempt: 3
03:46:25.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:25.194909 4583 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.97.252:45135: Network error: Client connection negotiation failed: client connection to 127.4.97.252:45135: connect: Connection refused (error 111)
03:46:25.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:25.335346 4583 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.97.252" port: 45135 } attempt: 4
03:46:25.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:25.339701 4583 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.97.252:45135: Network error: Client connection negotiation failed: client connection to 127.4.97.252:45135: connect: Connection refused (error 111)
03:46:25.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:25.623267 4583 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.97.252" port: 45135 } attempt: 5
03:46:25.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:25.627401 4583 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.97.252:45135: Network error: Client connection negotiation failed: client connection to 127.4.97.252:45135: connect: Connection refused (error 111)
03:46:25.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:25.726341 4662 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.97.252" port: 45135 } attempt: 6
03:46:25.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:25.733184 4662 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.97.252:45135: Network error: Client connection negotiation failed: client connection to 127.4.97.252:45135: connect: Connection refused (error 111)
03:46:26.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:24.888235 4680 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:46:26.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:24.889400 4668 server_base.cc:1034] running on GCE node
03:46:26.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:24.888289 4678 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:46:26.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:26.119151 4668 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:46:26.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:26.122241 4668 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:46:26.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:26.123660 4668 hybrid_clock.cc:648] HybridClock initialized: now 1790567186123629 us; error 48 us; skew 500 ppm
03:46:26.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:26.124390 4668 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:46:26.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 03:46:26 dist-test-slave-n4rz krb5kdc[4499](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790567186, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.97.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:46:26.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:26.130400 4668 init.cc:377] Logged in from keytab as kudu/127.4.97.252@KRBTEST.COM (short username kudu)
03:46:26.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:26.135864 4668 webserver.cc:460] Webserver started at http://127.4.97.252:46527/ using document root <none> and password file <none>
03:46:26.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:26.136704 4668 fs_manager.cc:359] Metadata directory not provided
03:46:26.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:26.136940 4668 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:46:26.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:26.137331 4668 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:46:26.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:26.144061 4668 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-2/data/instance:
03:46:26.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "30dba478be5b4cc9961b30e23b22414b"
03:46:26.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 03:46:26 on dist-test-slave-n4rz"
03:46:26.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:26.145612 4668 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-2/wal/instance:
03:46:26.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "30dba478be5b4cc9961b30e23b22414b"
03:46:26.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 03:46:26 on dist-test-slave-n4rz"
03:46:26.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:26.154731 4668 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.004s sys 0.004s
03:46:26.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:26.161957 4693 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:46:26.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:26.163297 4668 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
03:46:26.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:26.163678 4668 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-2/data,/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-2/wal
03:46:26.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "30dba478be5b4cc9961b30e23b22414b"
03:46:26.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 03:46:26 on dist-test-slave-n4rz"
03:46:26.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:26.164088 4668 fs_report.cc:383] FS layout report
03:46:26.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:46:26.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-2/wal
03:46:26.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-2/wal
03:46:26.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-2/data/data
03:46:26.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:46:26.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:46:26.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:46:26.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:46:26.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:46:26.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:46:26.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:46:26.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:46:26.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:46:26.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:46:26.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:46:26.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:26.179991 4583 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.97.252" port: 45135 } attempt: 6
03:46:26.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:26.207065 4583 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.97.252:45135: Network error: Client connection negotiation failed: client connection to 127.4.97.252:45135: connect: Connection refused (error 111)
03:46:26.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:26.221064 4668 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:46:26.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:26.227432 4668 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:46:26.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:26.227820 4668 kserver.cc:163] Server-wide thread pool size limit: 3276
03:46:26.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:26.296998 4668 rpc_server.cc:307] RPC server started. Bound to: 127.4.97.252:45135
03:46:26.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:26.297091 4746 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.97.252:45135 every 8 connection(s)
03:46:26.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:26.299777 4668 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-2/data/info.pb
03:46:26.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:26.305138 4487 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskbYfj1w/build/tsan/bin/kudu as pid 4668
03:46:26.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:26.305313 4747 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:46:26.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:26.305564 4487 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-2/wal/instance
03:46:26.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:26.319875 4747 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.97.254" port: 35997 } has no permanent_uuid. Determining permanent_uuid...
03:46:26.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 03:46:26 dist-test-slave-n4rz krb5kdc[4499](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790567186, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.97.252@KRBTEST.COM for kudu/127.4.97.254@KRBTEST.COM
03:46:26.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 03:46:26 dist-test-slave-n4rz krb5kdc[4499](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790567180, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.4.97.252@KRBTEST.COM
03:46:26.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:26.355672 4747 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.97.253" port: 46769 } has no permanent_uuid. Determining permanent_uuid...
03:46:26.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 03:46:26 dist-test-slave-n4rz krb5kdc[4499](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790567186, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.97.252@KRBTEST.COM for kudu/127.4.97.253@KRBTEST.COM
03:46:26.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 03:46:26 dist-test-slave-n4rz krb5kdc[4499](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790567180, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.4.97.253@KRBTEST.COM
03:46:26.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:26.383122 4747 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.97.252" port: 45135 } has no permanent_uuid. Determining permanent_uuid...
03:46:26.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 03:46:26 dist-test-slave-n4rz krb5kdc[4499](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790567186, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.97.252@KRBTEST.COM for kudu/127.4.97.252@KRBTEST.COM
03:46:26.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 03:46:26 dist-test-slave-n4rz krb5kdc[4499](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790567180, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.4.97.254@KRBTEST.COM
03:46:26.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:26.413563 4747 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b: Bootstrap starting.
03:46:26.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:26.418018 4487 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
03:46:26.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:26.419966 4747 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b: Neither blocks nor log segments found. Creating new log.
03:46:26.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:26.421623 4747 log.cc:824] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b: Log is configured to *not* fsync() on all Append() calls
03:46:26.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:26.426170 4747 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b: No bootstrap required, opened a new log
03:46:26.428 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:46:26.429 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
03:46:26.441 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
03:46:26.442 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
03:46:26.446 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.4.97.254"
port: 35997
}
bound_http_address {
host: "127.4.97.254"
port: 41607
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.4.97.253"
port: 46769
}
bound_http_address {
host: "127.4.97.253"
port: 41571
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.4.97.252"
port: 45135
}
bound_http_address {
host: "127.4.97.252"
port: 46527
}
}
}
03:46:26.447 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
03:46:26.447 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:46:26.447 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
03:46:26.448 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
03:46:26.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:26.452768 4747 raft_consensus.cc:348] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1639beaa478b4f928ca7167f7af13082" member_type: VOTER last_known_addr { host: "127.4.97.254" port: 35997 } } peers { permanent_uuid: "53a3a7a4248242bbbb5a044216cf0ae8" member_type: VOTER last_known_addr { host: "127.4.97.253" port: 46769 } } peers { permanent_uuid: "30dba478be5b4cc9961b30e23b22414b" member_type: VOTER last_known_addr { host: "127.4.97.252" port: 45135 } }
03:46:26.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:26.453488 4747 raft_consensus.cc:374] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:46:26.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:26.453809 4747 raft_consensus.cc:729] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 30dba478be5b4cc9961b30e23b22414b, State: Initialized, Role: FOLLOWER
03:46:26.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:26.454653 4747 consensus_queue.cc:260] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1639beaa478b4f928ca7167f7af13082" member_type: VOTER last_known_addr { host: "127.4.97.254" port: 35997 } } peers { permanent_uuid: "53a3a7a4248242bbbb5a044216cf0ae8" member_type: VOTER last_known_addr { host: "127.4.97.253" port: 46769 } } peers { permanent_uuid: "30dba478be5b4cc9961b30e23b22414b" member_type: VOTER last_known_addr { host: "127.4.97.252" port: 45135 } }
03:46:26.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:26.457777 4757 sys_catalog.cc:455] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1639beaa478b4f928ca7167f7af13082" member_type: VOTER last_known_addr { host: "127.4.97.254" port: 35997 } } peers { permanent_uuid: "53a3a7a4248242bbbb5a044216cf0ae8" member_type: VOTER last_known_addr { host: "127.4.97.253" port: 46769 } } peers { permanent_uuid: "30dba478be5b4cc9961b30e23b22414b" member_type: VOTER last_known_addr { host: "127.4.97.252" port: 45135 } } }
03:46:26.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:26.458881 4757 sys_catalog.cc:458] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [sys.catalog]: This master's current role is: FOLLOWER
03:46:26.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:26.459784 4747 sys_catalog.cc:564] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [sys.catalog]: configured and running, proceeding with master startup.
03:46:26.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:26.482710 4701 tablet.cc:2369] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:46:26.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 03:46:26 dist-test-slave-n4rz krb5kdc[4499](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790567186, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:46:26.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
03:46:26.494 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:46:26.496 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/krb5kdc/krb5cc
03:46:26.506 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
03:46:26.507 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/krb5kdc/krb5cc
03:46:26.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:26.506819 4769 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:46:26.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:26.507161 4769 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:46:26.508 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
03:46:26.515 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
03:46:26.517 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
03:46:26.518 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
03:46:26.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:26.484465 4668 master_runner.cc:186] Error getting master registration for 127.4.97.254:35997: OK, instance_id { permanent_uuid: "1639beaa478b4f928ca7167f7af13082" instance_seqno: 1790567182543282 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
03:46:26.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:26.525534 4668 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:46:26.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 03:46:26 dist-test-slave-n4rz krb5kdc[4499](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790567186, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:46:26.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
03:46:26.549 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:46:26.549 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/krb5kdc/krb5cc
03:46:26.551 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
03:46:26.552 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.4.97.254:35997
03:46:26.554 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
03:46:26.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:26.554562 4487 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskbYfj1w/build/tsan/bin/kudu with pid 4512
03:46:26.571 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:46:26.572 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.4.97.252:45135
03:46:26.573 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
03:46:26.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:26.573364 4487 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskbYfj1w/build/tsan/bin/kudu with pid 4668
03:46:26.593 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:46:26.593 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.4.97.253:46769
03:46:26.594 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
03:46:26.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:26.594530 4487 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskbYfj1w/build/tsan/bin/kudu with pid 4589
03:46:26.616 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:46:26.616 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.4.97.254:35997
03:46:26.618 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
03:46:26.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:26.619927 4487 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskbYfj1w/build/tsan/bin/kudu
03:46:26.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbYfj1w/build/tsan/bin/kudu
03:46:26.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-0/wal
03:46:26.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-0/data
03:46:26.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:46:26.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:46:26.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:46:26.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:46:26.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:46:26.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:46:26.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-0/logs
03:46:26.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-0/data/info.pb
03:46:26.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:46:26.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:46:26.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:46:26.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:46:26.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:46:26.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:46:26.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:46:26.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:46:26.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:46:26.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:46:26.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:46:26.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:46:26.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.97.254:35997
03:46:26.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.97.254
03:46:26.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41607
03:46:26.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/krb5kdc/kudu_127.4.97.254.keytab
03:46:26.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.97.254
03:46:26.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:46:26.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:46:26.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:46:26.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.97.212:40151
03:46:26.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:46:26.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:46:26.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:46:26.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:46:26.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.97.254:35997,127.4.97.253:46769,127.4.97.252:45135
03:46:26.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:46:26.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45935/jwks.json
03:46:26.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:46:26.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:26.925932 4773 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:46:26.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:26.926587 4773 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:46:26.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:26.927285 4773 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:46:26.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:26.970017 4773 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:46:26.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:26.970323 4773 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:46:26.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:26.970754 4773 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:46:26.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:26.970973 4773 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:46:26.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:26.971159 4773 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:46:26.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:26.971334 4773 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:46:26.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:26.971475 4773 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45935/jwks.json
03:46:27.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:27.009908 4773 master_runner.cc:386] Master server non-default flags:
03:46:27.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:46:27.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.97.212:40151
03:46:27.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:46:27.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:46:27.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-0/data
03:46:27.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-0/wal
03:46:27.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:46:27.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:46:27.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.97.254:35997,127.4.97.253:46769,127.4.97.252:45135
03:46:27.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/krb5kdc/kudu_127.4.97.254.keytab
03:46:27.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.97.254
03:46:27.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:46:27.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:46:27.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:46:27.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.97.254:35997
03:46:27.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:46:27.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:46:27.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:46:27.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45935/jwks.json
03:46:27.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:46:27.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:46:27.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:46:27.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:46:27.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:46:27.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-0/data/info.pb
03:46:27.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.97.254
03:46:27.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41607
03:46:27.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:46:27.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:46:27.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:46:27.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:46:27.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:46:27.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-0/logs
03:46:27.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:46:27.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:46:27.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:46:27.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:46:27.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:46:27.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:46:27.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:46:27.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 02:50:13 UTC on ccycloud.quasar-purgqw.root.comops.site
03:46:27.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676527
03:46:27.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:46:27.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:27.011224 4773 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:46:27.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:27.012348 4773 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:46:27.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:27.029955 4784 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:46:28.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:27.030200 4781 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:46:28.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:27.030798 4782 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:46:28.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:28.262216 4783 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
03:46:28.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:28.262562 4773 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:46:28.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:28.266117 4773 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:46:28.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:28.268467 4773 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:46:28.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:28.269855 4773 hybrid_clock.cc:648] HybridClock initialized: now 1790567188269814 us; error 42 us; skew 500 ppm
03:46:28.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:28.270701 4773 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:46:28.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 03:46:28 dist-test-slave-n4rz krb5kdc[4499](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790567188, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.97.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:46:28.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:28.276317 4773 init.cc:377] Logged in from keytab as kudu/127.4.97.254@KRBTEST.COM (short username kudu)
03:46:28.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:28.281682 4773 webserver.cc:460] Webserver started at http://127.4.97.254:41607/ using document root <none> and password file <none>
03:46:28.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:28.282743 4773 fs_manager.cc:359] Metadata directory not provided
03:46:28.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:28.282936 4773 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:46:28.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:28.290427 4773 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.001s sys 0.003s
03:46:28.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:28.294420 4792 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:46:28.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:28.295606 4773 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
03:46:28.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:28.295938 4773 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-0/data,/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-0/wal
03:46:28.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1639beaa478b4f928ca7167f7af13082"
03:46:28.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 03:46:22 on dist-test-slave-n4rz"
03:46:28.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:28.298293 4773 fs_report.cc:383] FS layout report
03:46:28.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:46:28.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-0/wal
03:46:28.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-0/wal
03:46:28.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-0/data/data
03:46:28.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:46:28.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:46:28.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:46:28.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:46:28.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:46:28.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:46:28.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:46:28.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:46:28.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:46:28.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:46:28.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:46:28.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:28.362561 4773 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:46:28.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:28.369534 4773 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:46:28.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:28.370083 4773 kserver.cc:163] Server-wide thread pool size limit: 3276
03:46:28.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:28.445848 4773 rpc_server.cc:307] RPC server started. Bound to: 127.4.97.254:35997
03:46:28.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:28.445981 4843 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.97.254:35997 every 8 connection(s)
03:46:28.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:28.448642 4773 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-0/data/info.pb
03:46:28.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:28.449797 4487 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskbYfj1w/build/tsan/bin/kudu as pid 4773
03:46:28.450 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:46:28.450 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.4.97.252:45135
03:46:28.451 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
03:46:28.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:28.452842 4487 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskbYfj1w/build/tsan/bin/kudu
03:46:28.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbYfj1w/build/tsan/bin/kudu
03:46:28.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-2/wal
03:46:28.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-2/data
03:46:28.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:46:28.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:46:28.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:46:28.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:46:28.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:46:28.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:46:28.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-2/logs
03:46:28.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-2/data/info.pb
03:46:28.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:46:28.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:46:28.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:46:28.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:46:28.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:46:28.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:46:28.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:46:28.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:46:28.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:46:28.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:46:28.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:46:28.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:46:28.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.97.252:45135
03:46:28.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.97.252
03:46:28.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46527
03:46:28.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/krb5kdc/kudu_127.4.97.252.keytab
03:46:28.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.97.252
03:46:28.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:46:28.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:46:28.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:46:28.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.97.212:40151
03:46:28.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:46:28.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:46:28.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:46:28.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:46:28.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.97.254:35997,127.4.97.253:46769,127.4.97.252:45135
03:46:28.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:46:28.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45935/jwks.json
03:46:28.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:46:28.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:28.462529 4844 sys_catalog.cc:263] Verifying existing consensus state
03:46:28.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260928 03:46:28.463459 4844 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
03:46:28.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:28.463819 4773 master.cc:537] Master@127.4.97.254:35997 shutting down...
03:46:28.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:28.526409 4773 master.cc:552] Master@127.4.97.254:35997 shutdown complete.
03:46:28.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
03:46:28.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:28.780648 4846 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:46:28.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:28.781272 4846 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:46:28.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:28.781863 4846 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:46:28.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:28.813750 4846 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:46:28.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:28.813964 4846 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:46:28.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:28.814237 4846 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:46:28.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:28.814541 4846 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:46:28.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:28.814846 4846 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:46:28.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:28.815231 4846 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:46:28.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:28.815474 4846 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45935/jwks.json
03:46:28.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:28.853639 4846 master_runner.cc:386] Master server non-default flags:
03:46:28.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:46:28.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.97.212:40151
03:46:28.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:46:28.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:46:28.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-2/data
03:46:28.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-2/wal
03:46:28.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:46:28.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:46:28.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.97.254:35997,127.4.97.253:46769,127.4.97.252:45135
03:46:28.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/krb5kdc/kudu_127.4.97.252.keytab
03:46:28.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.97.252
03:46:28.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:46:28.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:46:28.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:46:28.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.97.252:45135
03:46:28.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:46:28.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:46:28.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:46:28.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45935/jwks.json
03:46:28.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:46:28.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:46:28.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:46:28.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:46:28.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:46:28.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-2/data/info.pb
03:46:28.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.97.252
03:46:28.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46527
03:46:28.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:46:28.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:46:28.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:46:28.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:46:28.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:46:28.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-2/logs
03:46:28.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:46:28.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:46:28.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:46:28.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:46:28.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:46:28.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:46:28.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:46:28.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 02:50:13 UTC on ccycloud.quasar-purgqw.root.comops.site
03:46:28.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676527
03:46:28.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:46:28.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:28.854836 4846 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:46:28.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:28.856148 4846 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:46:28.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:28.875406 4856 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:46:30.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:28.875427 4854 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:46:30.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:28.875466 4853 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:46:30.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:28.876537 4846 server_base.cc:1034] running on GCE node
03:46:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:30.118671 4846 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:46:30.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:30.121402 4846 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:46:30.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:30.122780 4846 hybrid_clock.cc:648] HybridClock initialized: now 1790567190122754 us; error 41 us; skew 500 ppm
03:46:30.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:30.123651 4846 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:46:30.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 03:46:30 dist-test-slave-n4rz krb5kdc[4499](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790567190, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.97.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:46:30.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:30.134029 4846 init.cc:377] Logged in from keytab as kudu/127.4.97.252@KRBTEST.COM (short username kudu)
03:46:30.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:30.138528 4846 webserver.cc:460] Webserver started at http://127.4.97.252:46527/ using document root <none> and password file <none>
03:46:30.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:30.139421 4846 fs_manager.cc:359] Metadata directory not provided
03:46:30.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:30.139678 4846 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:46:30.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:30.149739 4846 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.007s sys 0.001s
03:46:30.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:30.154783 4865 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:46:30.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:30.155857 4846 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
03:46:30.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:30.156132 4846 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-2/data,/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-2/wal
03:46:30.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "30dba478be5b4cc9961b30e23b22414b"
03:46:30.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 03:46:26 on dist-test-slave-n4rz"
03:46:30.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:30.159147 4846 fs_report.cc:383] FS layout report
03:46:30.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:46:30.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-2/wal
03:46:30.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-2/wal
03:46:30.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-2/data/data
03:46:30.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:46:30.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:46:30.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:46:30.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:46:30.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:46:30.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:46:30.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:46:30.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:46:30.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:46:30.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:46:30.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:46:30.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:30.238545 4846 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:46:30.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:30.244128 4846 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:46:30.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:30.244522 4846 kserver.cc:163] Server-wide thread pool size limit: 3276
03:46:30.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:30.314144 4846 rpc_server.cc:307] RPC server started. Bound to: 127.4.97.252:45135
03:46:30.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:30.314294 4916 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.97.252:45135 every 8 connection(s)
03:46:30.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:30.316794 4846 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-2/data/info.pb
03:46:30.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:30.324661 4487 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskbYfj1w/build/tsan/bin/kudu as pid 4846
03:46:30.326 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:46:30.326 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.4.97.253:46769
03:46:30.326 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
03:46:30.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:30.327353 4917 sys_catalog.cc:263] Verifying existing consensus state
03:46:30.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:30.328023 4487 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskbYfj1w/build/tsan/bin/kudu
03:46:30.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbYfj1w/build/tsan/bin/kudu
03:46:30.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-1/wal
03:46:30.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-1/data
03:46:30.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:46:30.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:46:30.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:46:30.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:46:30.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:46:30.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:46:30.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-1/logs
03:46:30.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-1/data/info.pb
03:46:30.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:46:30.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:46:30.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:46:30.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:46:30.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:46:30.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:46:30.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:46:30.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:46:30.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:46:30.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:46:30.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:46:30.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:46:30.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.97.253:46769
03:46:30.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.97.253
03:46:30.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41571
03:46:30.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/krb5kdc/kudu_127.4.97.253.keytab
03:46:30.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.97.253
03:46:30.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:46:30.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:46:30.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:46:30.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.97.212:40151
03:46:30.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:46:30.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:46:30.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:46:30.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:46:30.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.97.254:35997,127.4.97.253:46769,127.4.97.252:45135
03:46:30.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:46:30.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45935/jwks.json
03:46:30.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:46:30.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:30.333225 4917 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b: Bootstrap starting.
03:46:30.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:30.377525 4917 log.cc:824] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b: Log is configured to *not* fsync() on all Append() calls
03:46:30.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:30.382473 4917 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
03:46:30.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:30.383317 4917 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b: Bootstrap complete.
03:46:30.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:30.415675 4917 raft_consensus.cc:348] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1639beaa478b4f928ca7167f7af13082" member_type: VOTER last_known_addr { host: "127.4.97.254" port: 35997 } } peers { permanent_uuid: "53a3a7a4248242bbbb5a044216cf0ae8" member_type: VOTER last_known_addr { host: "127.4.97.253" port: 46769 } } peers { permanent_uuid: "30dba478be5b4cc9961b30e23b22414b" member_type: VOTER last_known_addr { host: "127.4.97.252" port: 45135 } }
03:46:30.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:30.418833 4917 raft_consensus.cc:374] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:46:30.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:30.419183 4917 raft_consensus.cc:729] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 30dba478be5b4cc9961b30e23b22414b, State: Initialized, Role: FOLLOWER
03:46:30.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:30.420135 4917 consensus_queue.cc:260] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1639beaa478b4f928ca7167f7af13082" member_type: VOTER last_known_addr { host: "127.4.97.254" port: 35997 } } peers { permanent_uuid: "53a3a7a4248242bbbb5a044216cf0ae8" member_type: VOTER last_known_addr { host: "127.4.97.253" port: 46769 } } peers { permanent_uuid: "30dba478be5b4cc9961b30e23b22414b" member_type: VOTER last_known_addr { host: "127.4.97.252" port: 45135 } }
03:46:30.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:30.422792 4921 sys_catalog.cc:455] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1639beaa478b4f928ca7167f7af13082" member_type: VOTER last_known_addr { host: "127.4.97.254" port: 35997 } } peers { permanent_uuid: "53a3a7a4248242bbbb5a044216cf0ae8" member_type: VOTER last_known_addr { host: "127.4.97.253" port: 46769 } } peers { permanent_uuid: "30dba478be5b4cc9961b30e23b22414b" member_type: VOTER last_known_addr { host: "127.4.97.252" port: 45135 } } }
03:46:30.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:30.423638 4921 sys_catalog.cc:458] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [sys.catalog]: This master's current role is: FOLLOWER
03:46:30.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:30.424049 4917 sys_catalog.cc:564] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [sys.catalog]: configured and running, proceeding with master startup.
03:46:30.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:30.453063 4932 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:46:30.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:30.453696 4932 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:46:30.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:30.455885 4846 master_runner.cc:186] Error getting master registration for 127.4.97.254:35997: Network error: Client connection negotiation failed: client connection to 127.4.97.254:35997: connect: Connection refused (error 111),
03:46:30.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:30.476293 4846 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:46:30.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:30.499454 4871 tablet.cc:2369] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:46:30.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:30.683609 4919 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:46:30.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:30.684262 4919 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:46:30.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:30.685016 4919 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:46:30.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:30.687151 4921 raft_consensus.cc:482] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:46:30.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:30.687654 4921 raft_consensus.cc:504] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1639beaa478b4f928ca7167f7af13082" member_type: VOTER last_known_addr { host: "127.4.97.254" port: 35997 } } peers { permanent_uuid: "53a3a7a4248242bbbb5a044216cf0ae8" member_type: VOTER last_known_addr { host: "127.4.97.253" port: 46769 } } peers { permanent_uuid: "30dba478be5b4cc9961b30e23b22414b" member_type: VOTER last_known_addr { host: "127.4.97.252" port: 45135 } }
03:46:30.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:30.691811 4866 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.97.254:35997: connect: Connection refused (error 111)
03:46:30.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:30.692142 4921 leader_election.cc:288] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1639beaa478b4f928ca7167f7af13082 (127.4.97.254:35997), 53a3a7a4248242bbbb5a044216cf0ae8 (127.4.97.253:46769)
03:46:30.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:30.702522 4868 leader_election.cc:334] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 53a3a7a4248242bbbb5a044216cf0ae8 (127.4.97.253:46769): Network error: Client connection negotiation failed: client connection to 127.4.97.253:46769: connect: Connection refused (error 111)
03:46:30.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:30.703068 4866 leader_election.cc:334] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 1639beaa478b4f928ca7167f7af13082 (127.4.97.254:35997): Network error: Client connection negotiation failed: client connection to 127.4.97.254:35997: connect: Connection refused (error 111)
03:46:30.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:30.703616 4866 leader_election.cc:302] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 30dba478be5b4cc9961b30e23b22414b; no voters: 1639beaa478b4f928ca7167f7af13082, 53a3a7a4248242bbbb5a044216cf0ae8
03:46:30.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:30.704409 4921 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
03:46:30.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:30.726152 4919 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:46:30.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:30.726416 4919 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:46:30.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:30.726846 4919 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:46:30.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:30.727149 4919 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:46:30.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:30.727447 4919 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:46:30.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:30.727746 4919 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:46:30.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:30.727963 4919 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45935/jwks.json
03:46:30.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:30.763999 4919 master_runner.cc:386] Master server non-default flags:
03:46:30.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:46:30.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.97.212:40151
03:46:30.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:46:30.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:46:30.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-1/data
03:46:30.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-1/wal
03:46:30.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:46:30.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:46:30.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.97.254:35997,127.4.97.253:46769,127.4.97.252:45135
03:46:30.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/krb5kdc/kudu_127.4.97.253.keytab
03:46:30.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.97.253
03:46:30.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:46:30.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:46:30.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:46:30.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.97.253:46769
03:46:30.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:46:30.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:46:30.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:46:30.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45935/jwks.json
03:46:30.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:46:30.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:46:30.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:46:30.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:46:30.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:46:30.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-1/data/info.pb
03:46:30.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.97.253
03:46:30.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41571
03:46:30.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:46:30.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:46:30.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:46:30.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:46:30.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:46:30.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-1/logs
03:46:30.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:46:30.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:46:30.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:46:30.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:46:30.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:46:30.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:46:30.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:46:30.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 02:50:13 UTC on ccycloud.quasar-purgqw.root.comops.site
03:46:30.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676527
03:46:30.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:46:30.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:30.765540 4919 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:46:30.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:30.766731 4919 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:46:30.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:30.784970 4943 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:46:31.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:31.462210 4932 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:46:31.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:31.462699 4932 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:46:32.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:30.788226 4919 server_base.cc:1034] running on GCE node
03:46:32.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:30.786037 4945 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:46:32.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:30.786007 4942 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:46:32.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:32.066428 4919 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:46:32.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:32.069953 4919 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:46:32.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:32.071434 4919 hybrid_clock.cc:648] HybridClock initialized: now 1790567192071394 us; error 51 us; skew 500 ppm
03:46:32.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:32.072564 4919 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:46:32.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 03:46:32 dist-test-slave-n4rz krb5kdc[4499](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790567192, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.97.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:46:32.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:32.080416 4919 init.cc:377] Logged in from keytab as kudu/127.4.97.253@KRBTEST.COM (short username kudu)
03:46:32.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:32.089638 4919 webserver.cc:460] Webserver started at http://127.4.97.253:41571/ using document root <none> and password file <none>
03:46:32.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:32.090823 4919 fs_manager.cc:359] Metadata directory not provided
03:46:32.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:32.091099 4919 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:46:32.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:32.101257 4919 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.006s sys 0.001s
03:46:32.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:32.106534 4954 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:46:32.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:32.107638 4919 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.003s
03:46:32.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:32.107898 4919 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-1/data,/tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-1/wal
03:46:32.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "53a3a7a4248242bbbb5a044216cf0ae8"
03:46:32.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 03:46:24 on dist-test-slave-n4rz"
03:46:32.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:32.110172 4919 fs_report.cc:383] FS layout report
03:46:32.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:46:32.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-1/wal
03:46:32.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-1/wal
03:46:32.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-1/data/data
03:46:32.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:46:32.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:46:32.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:46:32.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:46:32.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:46:32.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:46:32.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:46:32.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:46:32.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:46:32.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:46:32.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:46:32.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:32.178730 4919 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:46:32.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:32.185099 4919 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:46:32.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:32.185617 4919 kserver.cc:163] Server-wide thread pool size limit: 3276
03:46:32.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:32.277648 4919 rpc_server.cc:307] RPC server started. Bound to: 127.4.97.253:46769
03:46:32.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:32.277823 5005 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.97.253:46769 every 8 connection(s)
03:46:32.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:32.280805 4919 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-1/data/info.pb
03:46:32.283 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:46:32.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:32.282140 4487 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskbYfj1w/build/tsan/bin/kudu as pid 4919
03:46:32.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:32.296241 5006 sys_catalog.cc:263] Verifying existing consensus state
03:46:32.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260928 03:46:32.297181 5006 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
03:46:32.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:32.297664 4919 master.cc:537] Master@127.4.97.253:46769 shutting down...
03:46:32.352 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.4.97.254:35997(127.4.97.254:35997)] connecting to peer
03:46:32.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:32.393899 4919 master.cc:552] Master@127.4.97.253:46769 shutdown complete.
03:46:32.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskbYfj1w/test-tmp/mini-kudu-cluster1754546088589048338/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
03:46:32.421 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.4.97.252:45135(127.4.97.252:45135)] connecting to peer
03:46:32.422 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.4.97.253:46769(127.4.97.253:46769)] connecting to peer
03:46:32.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:32.466990 4932 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:46:32.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:32.467522 4932 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:46:32.468 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.4.97.252:45135(127.4.97.252:45135)] Successfully connected to peer
03:46:32.472 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:32.473 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.97.254:35997(127.4.97.254:35997)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:32.477 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.97.254:35997: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:32.478 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:32.478 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.97.253:46769(127.4.97.253:46769)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:32.479 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.97.253:46769: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:32.550 [WARN - kudu-nio-1] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
03:46:32.558 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.4.97.252
03:46:32.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 03:46:32 dist-test-slave-n4rz krb5kdc[4499](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790567186, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.4.97.252@KRBTEST.COM
03:46:32.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:32.764596 5012 raft_consensus.cc:482] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:46:32.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:32.765545 5012 raft_consensus.cc:504] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1639beaa478b4f928ca7167f7af13082" member_type: VOTER last_known_addr { host: "127.4.97.254" port: 35997 } } peers { permanent_uuid: "53a3a7a4248242bbbb5a044216cf0ae8" member_type: VOTER last_known_addr { host: "127.4.97.253" port: 46769 } } peers { permanent_uuid: "30dba478be5b4cc9961b30e23b22414b" member_type: VOTER last_known_addr { host: "127.4.97.252" port: 45135 } }
03:46:32.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:32.767966 5012 leader_election.cc:288] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1639beaa478b4f928ca7167f7af13082 (127.4.97.254:35997), 53a3a7a4248242bbbb5a044216cf0ae8 (127.4.97.253:46769)
03:46:32.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:32.774632 4868 leader_election.cc:334] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 53a3a7a4248242bbbb5a044216cf0ae8 (127.4.97.253:46769): Network error: Client connection negotiation failed: client connection to 127.4.97.253:46769: connect: Connection refused (error 111)
03:46:32.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:32.775085 4866 leader_election.cc:334] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 1639beaa478b4f928ca7167f7af13082 (127.4.97.254:35997): Network error: Client connection negotiation failed: client connection to 127.4.97.254:35997: connect: Connection refused (error 111)
03:46:32.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:32.775540 4866 leader_election.cc:302] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 30dba478be5b4cc9961b30e23b22414b; no voters: 1639beaa478b4f928ca7167f7af13082, 53a3a7a4248242bbbb5a044216cf0ae8
03:46:32.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:32.776252 5012 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
03:46:32.804 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x2c4afabb, L:/127.0.0.1:36868 - R:/127.4.97.252:45135] using SASL/GSSAPI
03:46:32.820 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769; will retry
03:46:32.835 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.97.254:35997(127.4.97.254:35997)] connecting to peer
03:46:32.839 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:32.839 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.97.254:35997(127.4.97.254:35997)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:32.839 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.97.254:35997: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:32.840 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.97.253:46769(127.4.97.253:46769)] connecting to peer
03:46:32.842 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:32.842 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.97.253:46769(127.4.97.253:46769)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:32.842 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.97.253:46769: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:32.845 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769; will retry
03:46:32.853 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.97.254:35997(127.4.97.254:35997)] connecting to peer
03:46:32.855 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.97.253:46769(127.4.97.253:46769)] connecting to peer
03:46:32.858 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:32.859 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.97.253:46769(127.4.97.253:46769)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:32.859 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:32.859 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.97.253:46769: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:32.859 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.97.254:35997(127.4.97.254:35997)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:32.860 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.97.254:35997: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:32.863 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769; will retry
03:46:32.874 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.97.254:35997(127.4.97.254:35997)] connecting to peer
03:46:32.875 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.97.253:46769(127.4.97.253:46769)] connecting to peer
03:46:32.877 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:32.877 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.97.253:46769(127.4.97.253:46769)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:32.878 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.97.253:46769: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:32.879 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:32.879 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.97.254:35997(127.4.97.254:35997)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:32.880 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.97.254:35997: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:32.882 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769; will retry
03:46:32.893 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.97.254:35997(127.4.97.254:35997)] connecting to peer
03:46:32.895 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.97.253:46769(127.4.97.253:46769)] connecting to peer
03:46:32.897 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:32.897 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.97.253:46769(127.4.97.253:46769)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:32.898 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.97.253:46769: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:32.898 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:32.899 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.97.254:35997(127.4.97.254:35997)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:32.899 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.97.254:35997: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:32.900 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769; will retry
03:46:32.914 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.97.254:35997(127.4.97.254:35997)] connecting to peer
03:46:32.915 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.97.253:46769(127.4.97.253:46769)] connecting to peer
03:46:32.918 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:32.918 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.97.253:46769(127.4.97.253:46769)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:32.918 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.97.253:46769: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:32.922 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:32.922 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.97.254:35997(127.4.97.254:35997)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:32.922 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.97.254:35997: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:32.923 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769; will retry
03:46:32.973 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.97.254:35997(127.4.97.254:35997)] connecting to peer
03:46:32.975 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.97.253:46769(127.4.97.253:46769)] connecting to peer
03:46:32.977 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:32.977 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.97.253:46769(127.4.97.253:46769)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:32.978 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.97.253:46769: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:32.979 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:32.979 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.97.254:35997(127.4.97.254:35997)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:32.980 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.97.254:35997: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:32.982 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769; will retry
03:46:33.094 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.97.254:35997(127.4.97.254:35997)] connecting to peer
03:46:33.096 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.97.253:46769(127.4.97.253:46769)] connecting to peer
03:46:33.097 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:33.097 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:33.098 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.97.253:46769(127.4.97.253:46769)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:33.098 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.97.254:35997(127.4.97.254:35997)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:33.098 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.97.254:35997: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:33.098 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.97.253:46769: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:33.100 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769; will retry
03:46:33.134 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.97.254:35997(127.4.97.254:35997)] connecting to peer
03:46:33.136 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.97.253:46769(127.4.97.253:46769)] connecting to peer
03:46:33.137 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:33.137 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:33.138 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.97.253:46769(127.4.97.253:46769)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:33.138 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.97.254:35997(127.4.97.254:35997)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:33.138 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.97.253:46769: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:33.138 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.97.254:35997: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:33.140 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769; will retry
03:46:33.154 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.97.254:35997(127.4.97.254:35997)] connecting to peer
03:46:33.155 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.97.253:46769(127.4.97.253:46769)] connecting to peer
03:46:33.157 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:33.157 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.97.253:46769(127.4.97.253:46769)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:33.158 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.97.253:46769: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:33.158 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:33.158 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.97.254:35997(127.4.97.254:35997)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:33.159 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.97.254:35997: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:33.161 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769; will retry
03:46:33.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:33.470599 4932 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:46:33.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:33.470854 4932 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:46:34.154 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.97.254:35997(127.4.97.254:35997)] connecting to peer
03:46:34.155 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.97.253:46769(127.4.97.253:46769)] connecting to peer
03:46:34.157 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:34.157 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:34.157 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.97.254:35997(127.4.97.254:35997)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:34.157 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.97.253:46769(127.4.97.253:46769)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:34.158 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.97.254:35997: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:34.158 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.97.253:46769: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:34.159 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769; will retry
03:46:34.313 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.97.254:35997(127.4.97.254:35997)] connecting to peer
03:46:34.320 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.97.253:46769(127.4.97.253:46769)] connecting to peer
03:46:34.321 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:34.323 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.97.254:35997(127.4.97.254:35997)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:34.323 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.97.254:35997: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:34.323 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:34.323 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.97.253:46769(127.4.97.253:46769)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:34.324 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.97.253:46769: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:34.324 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769; will retry
03:46:34.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:34.473587 4932 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:46:34.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:34.473837 4932 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:46:35.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:35.476727 4932 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:46:35.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:35.477063 4932 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:46:35.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:35.478848 4846 master_runner.cc:186] Error getting master registration for 127.4.97.254:35997: Network error: Client connection negotiation failed: client connection to 127.4.97.254:35997: connect: Connection refused (error 111),
03:46:35.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:35.479241 4846 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:46:35.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:35.907300 5020 raft_consensus.cc:482] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:46:35.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:35.907668 5020 raft_consensus.cc:504] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1639beaa478b4f928ca7167f7af13082" member_type: VOTER last_known_addr { host: "127.4.97.254" port: 35997 } } peers { permanent_uuid: "53a3a7a4248242bbbb5a044216cf0ae8" member_type: VOTER last_known_addr { host: "127.4.97.253" port: 46769 } } peers { permanent_uuid: "30dba478be5b4cc9961b30e23b22414b" member_type: VOTER last_known_addr { host: "127.4.97.252" port: 45135 } }
03:46:35.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:35.909261 5020 leader_election.cc:288] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1639beaa478b4f928ca7167f7af13082 (127.4.97.254:35997), 53a3a7a4248242bbbb5a044216cf0ae8 (127.4.97.253:46769)
03:46:35.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:35.910481 4866 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.97.254:35997: connect: Connection refused (error 111) [suppressed 3 similar messages]
03:46:35.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:35.913547 4866 leader_election.cc:334] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 1639beaa478b4f928ca7167f7af13082 (127.4.97.254:35997): Network error: Client connection negotiation failed: client connection to 127.4.97.254:35997: connect: Connection refused (error 111)
03:46:35.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:35.914062 4868 leader_election.cc:334] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 53a3a7a4248242bbbb5a044216cf0ae8 (127.4.97.253:46769): Network error: Client connection negotiation failed: client connection to 127.4.97.253:46769: connect: Connection refused (error 111)
03:46:35.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:35.914366 4868 leader_election.cc:302] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 30dba478be5b4cc9961b30e23b22414b; no voters: 1639beaa478b4f928ca7167f7af13082, 53a3a7a4248242bbbb5a044216cf0ae8
03:46:35.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:35.914865 5020 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
03:46:36.113 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.97.254:35997(127.4.97.254:35997)] connecting to peer
03:46:36.115 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.97.253:46769(127.4.97.253:46769)] connecting to peer
03:46:36.116 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:36.116 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:36.116 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.97.253:46769(127.4.97.253:46769)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:36.117 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.97.254:35997(127.4.97.254:35997)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:36.117 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.97.253:46769: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:36.117 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.97.254:35997: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:36.119 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769; will retry
03:46:36.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:36.480031 4932 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:46:36.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:36.480279 4932 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:46:37.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:37.482976 4932 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:46:37.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:37.483227 4932 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:46:37.494 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.97.254:35997(127.4.97.254:35997)] connecting to peer
03:46:37.496 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.97.253:46769(127.4.97.253:46769)] connecting to peer
03:46:37.498 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:37.498 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:37.498 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.97.254:35997(127.4.97.254:35997)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:37.498 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.97.253:46769(127.4.97.253:46769)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:37.498 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.97.254:35997: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:37.498 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.97.253:46769: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:37.500 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769; will retry
03:46:38.434 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.97.254:35997(127.4.97.254:35997)] connecting to peer
03:46:38.436 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.97.253:46769(127.4.97.253:46769)] connecting to peer
03:46:38.438 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:38.438 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:38.438 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.97.254:35997(127.4.97.254:35997)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:38.438 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.97.253:46769(127.4.97.253:46769)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:38.438 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.97.254:35997: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:38.438 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.97.253:46769: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:38.440 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769; will retry
03:46:38.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:38.486088 4932 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:46:38.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:38.486357 4932 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:46:39.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:39.490085 4932 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:46:39.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:39.490518 4932 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:46:40.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:40.070884 5029 raft_consensus.cc:482] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:46:40.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:40.071383 5029 raft_consensus.cc:504] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1639beaa478b4f928ca7167f7af13082" member_type: VOTER last_known_addr { host: "127.4.97.254" port: 35997 } } peers { permanent_uuid: "53a3a7a4248242bbbb5a044216cf0ae8" member_type: VOTER last_known_addr { host: "127.4.97.253" port: 46769 } } peers { permanent_uuid: "30dba478be5b4cc9961b30e23b22414b" member_type: VOTER last_known_addr { host: "127.4.97.252" port: 45135 } }
03:46:40.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:40.073166 5029 leader_election.cc:288] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1639beaa478b4f928ca7167f7af13082 (127.4.97.254:35997), 53a3a7a4248242bbbb5a044216cf0ae8 (127.4.97.253:46769)
03:46:40.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:40.079097 4866 leader_election.cc:334] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 1639beaa478b4f928ca7167f7af13082 (127.4.97.254:35997): Network error: Client connection negotiation failed: client connection to 127.4.97.254:35997: connect: Connection refused (error 111)
03:46:40.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:40.079794 4868 leader_election.cc:334] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 53a3a7a4248242bbbb5a044216cf0ae8 (127.4.97.253:46769): Network error: Client connection negotiation failed: client connection to 127.4.97.253:46769: connect: Connection refused (error 111)
03:46:40.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:40.080140 4868 leader_election.cc:302] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 30dba478be5b4cc9961b30e23b22414b; no voters: 1639beaa478b4f928ca7167f7af13082, 53a3a7a4248242bbbb5a044216cf0ae8
03:46:40.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:40.080922 5029 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
03:46:40.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 03:46:40 dist-test-slave-n4rz krb5kdc[4499](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790567200, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.97.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:46:40.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:40.137727 4861 init.cc:295] Successfully reacquired a new kerberos TGT
03:46:40.394 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.97.254:35997(127.4.97.254:35997)] connecting to peer
03:46:40.396 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.97.253:46769(127.4.97.253:46769)] connecting to peer
03:46:40.398 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:40.398 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:40.398 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.97.254:35997(127.4.97.254:35997)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:40.398 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.97.253:46769(127.4.97.253:46769)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:40.398 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.97.254:35997: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:40.398 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.97.253:46769: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:40.400 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769; will retry
03:46:40.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:40.481407 4846 master_runner.cc:186] Error getting master registration for 127.4.97.254:35997: Network error: Client connection negotiation failed: client connection to 127.4.97.254:35997: connect: Connection refused (error 111),
03:46:40.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:40.481804 4846 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:46:40.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:40.493350 4932 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:46:40.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:40.493592 4932 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:46:41.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:41.496611 4932 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:46:41.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:41.496882 4932 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:46:42.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:42.499570 4932 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:46:42.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:42.499816 4932 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:46:43.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:43.502779 4932 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:46:43.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:43.503039 4932 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:46:44.434 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.97.254:35997(127.4.97.254:35997)] connecting to peer
03:46:44.435 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.97.253:46769(127.4.97.253:46769)] connecting to peer
03:46:44.436 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:44.436 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:44.437 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.97.253:46769(127.4.97.253:46769)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:44.437 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.97.254:35997(127.4.97.254:35997)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:44.437 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.97.253:46769: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:44.437 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.97.254:35997: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:44.439 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769; will retry
03:46:44.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:44.505880 4932 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:46:44.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:44.506127 4932 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:46:44.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:44.651998 5035 raft_consensus.cc:482] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:46:44.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:44.652347 5035 raft_consensus.cc:504] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1639beaa478b4f928ca7167f7af13082" member_type: VOTER last_known_addr { host: "127.4.97.254" port: 35997 } } peers { permanent_uuid: "53a3a7a4248242bbbb5a044216cf0ae8" member_type: VOTER last_known_addr { host: "127.4.97.253" port: 46769 } } peers { permanent_uuid: "30dba478be5b4cc9961b30e23b22414b" member_type: VOTER last_known_addr { host: "127.4.97.252" port: 45135 } }
03:46:44.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:44.653908 5035 leader_election.cc:288] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1639beaa478b4f928ca7167f7af13082 (127.4.97.254:35997), 53a3a7a4248242bbbb5a044216cf0ae8 (127.4.97.253:46769)
03:46:44.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:44.656819 4866 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.97.254:35997: connect: Connection refused (error 111) [suppressed 3 similar messages]
03:46:44.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:44.659864 4866 leader_election.cc:334] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 1639beaa478b4f928ca7167f7af13082 (127.4.97.254:35997): Network error: Client connection negotiation failed: client connection to 127.4.97.254:35997: connect: Connection refused (error 111)
03:46:44.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:44.660231 4868 leader_election.cc:334] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 53a3a7a4248242bbbb5a044216cf0ae8 (127.4.97.253:46769): Network error: Client connection negotiation failed: client connection to 127.4.97.253:46769: connect: Connection refused (error 111)
03:46:44.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:44.660619 4868 leader_election.cc:302] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 30dba478be5b4cc9961b30e23b22414b; no voters: 1639beaa478b4f928ca7167f7af13082, 53a3a7a4248242bbbb5a044216cf0ae8
03:46:44.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:44.661285 5035 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
03:46:45.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:45.484273 4846 master_runner.cc:186] Error getting master registration for 127.4.97.254:35997: Network error: Client connection negotiation failed: client connection to 127.4.97.254:35997: connect: Connection refused (error 111),
03:46:45.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:45.484587 4846 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:46:45.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:45.509485 4932 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:46:45.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:45.509783 4932 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:46:46.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:46.512547 4932 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:46:46.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:46.512846 4932 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:46:47.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:47.515971 4932 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:46:47.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:47.516228 4932 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:46:48.494 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.97.254:35997(127.4.97.254:35997)] connecting to peer
03:46:48.496 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.97.253:46769(127.4.97.253:46769)] connecting to peer
03:46:48.498 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:48.498 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.97.253:46769(127.4.97.253:46769)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:48.498 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.97.253:46769: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:48.498 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:48.498 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.97.254:35997(127.4.97.254:35997)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:48.498 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.97.254:35997: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:48.500 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769; will retry
03:46:48.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:48.519125 4932 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:46:48.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:48.519423 4932 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:46:49.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:49.522925 4932 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:46:49.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:49.523263 4932 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:46:49.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:49.730520 5047 raft_consensus.cc:482] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:46:49.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:49.730978 5047 raft_consensus.cc:504] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1639beaa478b4f928ca7167f7af13082" member_type: VOTER last_known_addr { host: "127.4.97.254" port: 35997 } } peers { permanent_uuid: "53a3a7a4248242bbbb5a044216cf0ae8" member_type: VOTER last_known_addr { host: "127.4.97.253" port: 46769 } } peers { permanent_uuid: "30dba478be5b4cc9961b30e23b22414b" member_type: VOTER last_known_addr { host: "127.4.97.252" port: 45135 } }
03:46:49.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:49.732923 5047 leader_election.cc:288] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1639beaa478b4f928ca7167f7af13082 (127.4.97.254:35997), 53a3a7a4248242bbbb5a044216cf0ae8 (127.4.97.253:46769)
03:46:49.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:49.735808 4866 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.97.254:35997: connect: Connection refused (error 111) [suppressed 1 similar messages]
03:46:49.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:49.738953 4866 leader_election.cc:334] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 1639beaa478b4f928ca7167f7af13082 (127.4.97.254:35997): Network error: Client connection negotiation failed: client connection to 127.4.97.254:35997: connect: Connection refused (error 111)
03:46:49.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:49.739969 4868 leader_election.cc:334] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 53a3a7a4248242bbbb5a044216cf0ae8 (127.4.97.253:46769): Network error: Client connection negotiation failed: client connection to 127.4.97.253:46769: connect: Connection refused (error 111)
03:46:49.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:49.740341 4868 leader_election.cc:302] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 30dba478be5b4cc9961b30e23b22414b; no voters: 1639beaa478b4f928ca7167f7af13082, 53a3a7a4248242bbbb5a044216cf0ae8
03:46:49.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:49.741043 5047 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
03:46:50.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 03:46:50 dist-test-slave-n4rz krb5kdc[4499](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790567210, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.97.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:46:50.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:50.140312 4861 init.cc:295] Successfully reacquired a new kerberos TGT
03:46:50.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:50.487154 4846 master_runner.cc:186] Error getting master registration for 127.4.97.254:35997: Network error: Client connection negotiation failed: client connection to 127.4.97.254:35997: connect: Connection refused (error 111),
03:46:50.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:50.487582 4846 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:46:50.494 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.97.254:35997(127.4.97.254:35997)] connecting to peer
03:46:50.495 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.97.253:46769(127.4.97.253:46769)] connecting to peer
03:46:50.498 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:50.498 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.97.253:46769(127.4.97.253:46769)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:50.498 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.97.253:46769: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:50.499 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:50.499 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.97.254:35997(127.4.97.254:35997)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:50.499 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.97.254:35997: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:50.499 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769; will retry
03:46:50.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:50.526180 4932 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:46:50.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:50.526475 4932 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:46:51.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:51.529421 4932 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:46:51.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:51.529681 4932 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:46:52.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:52.532362 4932 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:46:52.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:52.532599 4932 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:46:53.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:53.535266 4932 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:46:53.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:53.535509 4932 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:46:53.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:53.827343 5053 raft_consensus.cc:482] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:46:53.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:53.827752 5053 raft_consensus.cc:504] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1639beaa478b4f928ca7167f7af13082" member_type: VOTER last_known_addr { host: "127.4.97.254" port: 35997 } } peers { permanent_uuid: "53a3a7a4248242bbbb5a044216cf0ae8" member_type: VOTER last_known_addr { host: "127.4.97.253" port: 46769 } } peers { permanent_uuid: "30dba478be5b4cc9961b30e23b22414b" member_type: VOTER last_known_addr { host: "127.4.97.252" port: 45135 } }
03:46:53.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:53.829340 5053 leader_election.cc:288] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1639beaa478b4f928ca7167f7af13082 (127.4.97.254:35997), 53a3a7a4248242bbbb5a044216cf0ae8 (127.4.97.253:46769)
03:46:53.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:53.833621 4866 leader_election.cc:334] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 1639beaa478b4f928ca7167f7af13082 (127.4.97.254:35997): Network error: Client connection negotiation failed: client connection to 127.4.97.254:35997: connect: Connection refused (error 111)
03:46:53.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:53.834038 4868 leader_election.cc:334] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 53a3a7a4248242bbbb5a044216cf0ae8 (127.4.97.253:46769): Network error: Client connection negotiation failed: client connection to 127.4.97.253:46769: connect: Connection refused (error 111)
03:46:53.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:53.834394 4868 leader_election.cc:302] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 30dba478be5b4cc9961b30e23b22414b; no voters: 1639beaa478b4f928ca7167f7af13082, 53a3a7a4248242bbbb5a044216cf0ae8
03:46:53.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:53.834987 5053 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
03:46:54.193 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.97.254:35997(127.4.97.254:35997)] connecting to peer
03:46:54.195 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.97.253:46769(127.4.97.253:46769)] connecting to peer
03:46:54.196 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:54.196 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:54.197 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.97.253:46769(127.4.97.253:46769)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:54.197 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.97.254:35997(127.4.97.254:35997)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:54.197 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.97.253:46769: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:54.197 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.97.254:35997: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:54.199 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769; will retry
03:46:54.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:54.538143 4932 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:46:54.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:54.538467 4932 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:46:54.954 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.97.254:35997(127.4.97.254:35997)] connecting to peer
03:46:54.955 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.97.253:46769(127.4.97.253:46769)] connecting to peer
03:46:54.956 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:54.956 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.97.254:35997(127.4.97.254:35997)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:54.956 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.97.254:35997: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:54.957 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:54.957 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.97.253:46769(127.4.97.253:46769)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:54.957 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.97.253:46769: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:54.959 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769; will retry
03:46:55.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:55.490633 4846 master_runner.cc:186] Error getting master registration for 127.4.97.254:35997: Network error: Client connection negotiation failed: client connection to 127.4.97.254:35997: connect: Connection refused (error 111),
03:46:55.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:55.491046 4846 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:46:55.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:55.541268 4932 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:46:55.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:55.541517 4932 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:46:55.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:55.583799 5061 raft_consensus.cc:482] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:46:55.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:55.584174 5061 raft_consensus.cc:504] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1639beaa478b4f928ca7167f7af13082" member_type: VOTER last_known_addr { host: "127.4.97.254" port: 35997 } } peers { permanent_uuid: "53a3a7a4248242bbbb5a044216cf0ae8" member_type: VOTER last_known_addr { host: "127.4.97.253" port: 46769 } } peers { permanent_uuid: "30dba478be5b4cc9961b30e23b22414b" member_type: VOTER last_known_addr { host: "127.4.97.252" port: 45135 } }
03:46:55.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:55.585702 5061 leader_election.cc:288] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1639beaa478b4f928ca7167f7af13082 (127.4.97.254:35997), 53a3a7a4248242bbbb5a044216cf0ae8 (127.4.97.253:46769)
03:46:55.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:55.586644 4866 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.97.254:35997: connect: Connection refused (error 111) [suppressed 3 similar messages]
03:46:55.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:55.589574 4866 leader_election.cc:334] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 1639beaa478b4f928ca7167f7af13082 (127.4.97.254:35997): Network error: Client connection negotiation failed: client connection to 127.4.97.254:35997: connect: Connection refused (error 111)
03:46:55.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:55.590096 4868 leader_election.cc:334] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 53a3a7a4248242bbbb5a044216cf0ae8 (127.4.97.253:46769): Network error: Client connection negotiation failed: client connection to 127.4.97.253:46769: connect: Connection refused (error 111)
03:46:55.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:55.590384 4868 leader_election.cc:302] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 30dba478be5b4cc9961b30e23b22414b; no voters: 1639beaa478b4f928ca7167f7af13082, 53a3a7a4248242bbbb5a044216cf0ae8
03:46:55.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:46:55.590857 5061 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
03:46:56.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:56.544153 4932 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:46:56.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:56.544449 4932 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:46:56.954 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.97.254:35997(127.4.97.254:35997)] connecting to peer
03:46:56.955 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.97.253:46769(127.4.97.253:46769)] connecting to peer
03:46:56.956 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:56.956 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:56.956 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.97.254:35997(127.4.97.254:35997)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:56.956 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.97.253:46769(127.4.97.253:46769)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:56.956 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.97.253:46769: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:56.956 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.97.254:35997: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:56.958 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769; will retry
03:46:57.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:57.547526 4932 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:46:57.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:57.547804 4932 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:46:57.934 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.97.254:35997(127.4.97.254:35997)] connecting to peer
03:46:57.935 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.97.253:46769(127.4.97.253:46769)] connecting to peer
03:46:57.935 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:57.936 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:57.936 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.97.254:35997(127.4.97.254:35997)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:57.936 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.97.253:46769(127.4.97.253:46769)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:57.936 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.97.254:35997: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:46:57.936 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.97.253:46769: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:46:57.938 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769; will retry
03:46:58.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:58.550771 4932 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:46:58.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:58.551088 4932 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:46:59.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:59.554667 4932 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 30dba478be5b4cc9961b30e23b22414b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:46:59.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:46:59.555071 4932 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:47:00.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 03:47:00 dist-test-slave-n4rz krb5kdc[4499](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790567220, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.97.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:47:00.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:47:00.142879 4861 init.cc:295] Successfully reacquired a new kerberos TGT
03:47:00.214 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.97.254:35997(127.4.97.254:35997)] connecting to peer
03:47:00.215 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.97.253:46769(127.4.97.253:46769)] connecting to peer
03:47:00.216 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:47:00.216 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:47:00.216 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.97.254:35997(127.4.97.254:35997)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:47:00.216 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.97.253:46769(127.4.97.253:46769)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:47:00.216 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.97.254:35997: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:47:00.217 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.97.253:46769: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:47:00.219 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769; will retry
03:47:00.454 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.97.254:35997(127.4.97.254:35997)] connecting to peer
03:47:00.455 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.97.253:46769(127.4.97.253:46769)] connecting to peer
03:47:00.456 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:47:00.456 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:47:00.456 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.97.253:46769(127.4.97.253:46769)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:47:00.456 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.97.254:35997(127.4.97.254:35997)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:47:00.456 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.97.253:46769: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769
03:47:00.456 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.97.254:35997: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
03:47:00.458 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769; will retry
03:47:00.463 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=28171), Traces: [0ms] refreshing cache from master, [45ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.254:35997, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.252:45135, [123ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.253:46769, [175ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.254:35997: Network error: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [181ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.253:46769: Network error: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [518ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.252:45135: OK, [522ms] delaying RPC due to: Service unavailable: Master config (127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [536ms] refreshing cache from master, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.254:35997, [538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.252:45135, [541ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.254:35997: Network error: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [541ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.253:46769, [544ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.253:46769: Network error: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [547ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.252:45135: OK, [548ms] delaying RPC due to: Service unavailable: Master config (127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [555ms] refreshing cache from master, [555ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.254:35997, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.252:45135, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.253:46769, [561ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.253:46769: Network error: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [561ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.254:35997: Network error: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [563ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.252:45135: OK, [565ms] delaying RPC due to: Service unavailable: Master config (127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [575ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.254:35997, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.252:45135, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.253:46769, [580ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.253:46769: Network error: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [581ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.254:35997: Network error: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [583ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.252:45135: OK, [584ms] delaying RPC due to: Service unavailable: Master config (127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [595ms] refreshing cache from master, [595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.254:35997, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.252:45135, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.253:46769, [599ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.253:46769: Network error: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [601ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.254:35997: Network error: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [602ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.252:45135: OK, [603ms] delaying RPC due to: Service unavailable: Master config (127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [615ms] refreshing cache from master, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.254:35997, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.252:45135, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.253:46769, [620ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.253:46769: Network error: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [623ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.252:45135: OK, [624ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.254:35997: Network error: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [625ms] delaying RPC due to: Service unavailable: Master config (127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [675ms] refreshing cache from master, [675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.254:35997, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.252:45135, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.253:46769, [680ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.253:46769: Network error: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [681ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.254:35997: Network error: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [683ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.252:45135: OK, [684ms] delaying RPC due to: Service unavailable: Master config (127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [795ms] refreshing cache from master, [796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.254:35997, [797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.252:45135, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.253:46769, [800ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.253:46769: Network error: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [800ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.254:35997: Network error: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [802ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.252:45135: OK, [802ms] delaying RPC due to: Service unavailable: Master config (127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [836ms] refreshing cache from master, [836ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.254:35997, [837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.252:45135, [837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.253:46769, [840ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.253:46769: Network error: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [840ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.254:35997: Network error: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [841ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.252:45135: OK, [842ms] delaying RPC due to: Service unavailable: Master config (127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [855ms] refreshing cache from master, [855ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.254:35997, [857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.252:45135, [857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.253:46769, [859ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.253:46769: Network error: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [861ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.254:35997: Network error: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [862ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.252:45135: OK, [863ms] delaying RPC due to: Service unavailable: Master config (127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [1855ms] refreshing cache from master, [1856ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.254:35997, [1857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.252:45135, [1857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.253:46769, [1859ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.254:35997: Network error: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [1860ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.253:46769: Network error: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [1861ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.252:45135: OK, [1861ms] delaying RPC due to: Service unavailable: Master config (127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [2015ms] refreshing cache from master, [2015ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.254:35997, [2016ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.252:45135, [2022ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.253:46769, [2025ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.254:35997: Network error: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [2025ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.253:46769: Network error: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [2026ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.252:45135: OK, [2026ms] delaying RPC due to: Service unavailable: Master config (127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [3815ms] refreshing cache from master, [3815ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.254:35997, [3816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.252:45135, [3816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.253:46769, [3819ms] too many traces: truncated at 100 traces, deferred=Deferred@1871594101(state=PENDING, result=null, callback=(continuation of Deferred@1929086179 after retry RPC after error@1286887968) -> (continuation of Deferred@1979264459 after retry RPC after error@1752782059) -> (continuation of Deferred@1183521179 after retry RPC after error@679489072) -> (continuation of Deferred@346802590 after retry RPC after error@920062444) -> (continuation of Deferred@680429932 after retry RPC after error@22467555) -> (continuation of Deferred@1266355566 after retry RPC after error@1164830084) -> (continuation of Deferred@435470764 after retry RPC after error@780845242) -> (continuation of Deferred@1032435832 after retry RPC after error@400698248) -> (continuation of Deferred@681861497 after retry RPC after error@1356069831) -> (continuation of Deferred@543806429 after retry RPC after error@642843187) -> (continuation of Deferred@771035177 after retry RPC after error@1656212371) -> (continuation of Deferred@204348195 after retry RPC after error@574672911) -> (continuation of Deferred@657582134 after retry RPC after error@2072396816) -> (continuation of Deferred@126725952 after retry RPC after error@602523414) -> (continuation of Deferred@196213509 after retry RPC after error@1121880231) -> (continuation of Deferred@675534867 after retry RPC after error@1165982586) -> (continuation of Deferred@1091754243 after retry RPC after error@2000677430) -> (continuation of Deferred@1099573122 after retry RPC after error@2131763298) -> (continuation of Deferred@681391170 after retry RPC after error@544933181) -> (continuation of Deferred@819826823 after retry RPC after error@1311811022) -> (continuation of Deferred@265205614 after retry RPC after error@2011437383) -> (continuation of Deferred@736229012 after retry RPC after error@1614031091) -> (continuation of Deferred@448501340 after retry RPC after error@1296206445) -> (continuation of Deferred@1773371399 after retry RPC after error@671178354), errback=(continuation of Deferred@1929086179 after retry RPC after error@1286887968) -> (continuation of Deferred@1979264459 after retry RPC after error@1752782059) -> (continuation of Deferred@1183521179 after retry RPC after error@679489072) -> (continuation of Deferred@346802590 after retry RPC after error@920062444) -> (continuation of Deferred@680429932 after retry RPC after error@22467555) -> (continuation of Deferred@1266355566 after retry RPC after error@1164830084) -> (continuation of Deferred@435470764 after retry RPC after error@780845242) -> (continuation of Deferred@1032435832 after retry RPC after error@400698248) -> (continuation of Deferred@681861497 after retry RPC after error@1356069831) -> (continuation of Deferred@543806429 after retry RPC after error@642843187) -> (continuation of Deferred@771035177 after retry RPC after error@1656212371) -> (continuation of Deferred@204348195 after retry RPC after error@574672911) -> (continuation of Deferred@657582134 after retry RPC after error@2072396816) -> (continuation of Deferred@126725952 after retry RPC after error@602523414) -> (continuation of Deferred@196213509 after retry RPC after error@1121880231) -> (continuation of Deferred@675534867 after retry RPC after error@1165982586) -> (continuation of Deferred@1091754243 after retry RPC after error@2000677430) -> (continuation of Deferred@1099573122 after retry RPC after error@2131763298) -> (continuation of Deferred@681391170 after retry RPC after error@544933181) -> (continuation of Deferred@819826823 after retry RPC after error@1311811022) -> (continuation of Deferred@265205614 after retry RPC after error@2011437383) -> (continuation of Deferred@736229012 after retry RPC after error@1614031091) -> (continuation of Deferred@448501340 after retry RPC after error@1296206445) -> (continuation of Deferred@1773371399 after retry RPC after error@671178354)))
03:47:00.468 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.4.97.253:46769(127.4.97.253:46769)] disconnecting while in state TERMINATED
03:47:00.469 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.4.97.252:45135(127.4.97.252:45135)] disconnecting while in state READY
03:47:00.469 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.4.97.254:35997(127.4.97.254:35997)] disconnecting while in state TERMINATED
03:47:00.474 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
03:47:00.476 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
03:47:00.477 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.4.97.252:45135(127.4.97.252:45135)] handling channelInactive
03:47:00.484 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.97.252:45135(127.4.97.252:45135)] cleaning up while in state READY due to: connection closed
03:47:00.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:47:00.484881 4487 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskbYfj1w/build/tsan/bin/kudu with pid 4846
03:47:01.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-28T03:47:01Z chronyd exiting
03:47:02.675 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
03:47:02.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:47:02.676684 3758 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskbYfj1w/build/tsan/bin/kudu with pid 4038
03:47:02.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:47:02.703029 3758 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskbYfj1w/build/tsan/bin/kudu with pid 4203
03:47:02.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:47:02.730841 3758 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskbYfj1w/build/tsan/bin/kudu with pid 4345
03:47:02.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:47:02.754784 3758 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskbYfj1w/build/tsan/bin/kudu with pid 3770
03:47:02.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:47:02.783041 3758 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskbYfj1w/build/tsan/bin/kudu with pid 3840
03:47:02.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:47:02.811215 3758 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskbYfj1w/build/tsan/bin/kudu with pid 3915
03:47:03.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-28T03:47:03Z chronyd exiting
03:47:04.987 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=28171), Traces: [0ms] refreshing cache from master, [45ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.254:35997, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.252:45135, [123ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.253:46769, [175ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.254:35997: Network error: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [181ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.253:46769: Network error: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [518ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.252:45135: OK, [522ms] delaying RPC due to: Service unavailable: Master config (127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [536ms] refreshing cache from master, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.254:35997, [538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.252:45135, [541ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.254:35997: Network error: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [541ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.253:46769, [544ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.253:46769: Network error: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [547ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.252:45135: OK, [548ms] delaying RPC due to: Service unavailable: Master config (127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [555ms] refreshing cache from master, [555ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.254:35997, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.252:45135, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.253:46769, [561ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.253:46769: Network error: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [561ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.254:35997: Network error: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [563ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.252:45135: OK, [565ms] delaying RPC due to: Service unavailable: Master config (127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [575ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.254:35997, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.252:45135, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.253:46769, [580ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.253:46769: Network error: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [581ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.254:35997: Network error: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [583ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.252:45135: OK, [584ms] delaying RPC due to: Service unavailable: Master config (127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [595ms] refreshing cache from master, [595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.254:35997, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.252:45135, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.253:46769, [599ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.253:46769: Network error: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [601ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.254:35997: Network error: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [602ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.252:45135: OK, [603ms] delaying RPC due to: Service unavailable: Master config (127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [615ms] refreshing cache from master, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.254:35997, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.252:45135, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.253:46769, [620ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.253:46769: Network error: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [623ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.252:45135: OK, [624ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.254:35997: Network error: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [625ms] delaying RPC due to: Service unavailable: Master config (127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [675ms] refreshing cache from master, [675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.254:35997, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.252:45135, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.253:46769, [680ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.253:46769: Network error: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [681ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.254:35997: Network error: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [683ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.252:45135: OK, [684ms] delaying RPC due to: Service unavailable: Master config (127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [795ms] refreshing cache from master, [796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.254:35997, [797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.252:45135, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.253:46769, [800ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.253:46769: Network error: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [800ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.254:35997: Network error: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [802ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.252:45135: OK, [802ms] delaying RPC due to: Service unavailable: Master config (127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [836ms] refreshing cache from master, [836ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.254:35997, [837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.252:45135, [837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.253:46769, [840ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.253:46769: Network error: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [840ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.254:35997: Network error: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [841ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.252:45135: OK, [842ms] delaying RPC due to: Service unavailable: Master config (127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [855ms] refreshing cache from master, [855ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.254:35997, [857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.252:45135, [857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.253:46769, [859ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.253:46769: Network error: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [861ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.254:35997: Network error: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [862ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.252:45135: OK, [863ms] delaying RPC due to: Service unavailable: Master config (127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [1855ms] refreshing cache from master, [1856ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.254:35997, [1857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.252:45135, [1857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.253:46769, [1859ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.254:35997: Network error: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [1860ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.253:46769: Network error: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [1861ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.252:45135: OK, [1861ms] delaying RPC due to: Service unavailable: Master config (127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [2015ms] refreshing cache from master, [2015ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.254:35997, [2016ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.252:45135, [2022ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.253:46769, [2025ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.254:35997: Network error: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [2025ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.253:46769: Network error: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [2026ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.252:45135: OK, [2026ms] delaying RPC due to: Service unavailable: Master config (127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [3815ms] refreshing cache from master, [3815ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.254:35997, [3816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.252:45135, [3816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.253:46769, [3819ms] too many traces: truncated at 100 traces, deferred=Deferred@1871594101(state=PENDING, result=null, callback=(continuation of Deferred@1929086179 after retry RPC after error@1286887968) -> (continuation of Deferred@1979264459 after retry RPC after error@1752782059) -> (continuation of Deferred@1183521179 after retry RPC after error@679489072) -> (continuation of Deferred@346802590 after retry RPC after error@920062444) -> (continuation of Deferred@680429932 after retry RPC after error@22467555) -> (continuation of Deferred@1266355566 after retry RPC after error@1164830084) -> (continuation of Deferred@435470764 after retry RPC after error@780845242) -> (continuation of Deferred@1032435832 after retry RPC after error@400698248) -> (continuation of Deferred@681861497 after retry RPC after error@1356069831) -> (continuation of Deferred@543806429 after retry RPC after error@642843187) -> (continuation of Deferred@771035177 after retry RPC after error@1656212371) -> (continuation of Deferred@204348195 after retry RPC after error@574672911) -> (continuation of Deferred@657582134 after retry RPC after error@2072396816) -> (continuation of Deferred@126725952 after retry RPC after error@602523414) -> (continuation of Deferred@196213509 after retry RPC after error@1121880231) -> (continuation of Deferred@675534867 after retry RPC after error@1165982586) -> (continuation of Deferred@1091754243 after retry RPC after error@2000677430) -> (continuation of Deferred@1099573122 after retry RPC after error@2131763298) -> (continuation of Deferred@681391170 after retry RPC after error@544933181) -> (continuation of Deferred@819826823 after retry RPC after error@1311811022) -> (continuation of Deferred@265205614 after retry RPC after error@2011437383) -> (continuation of Deferred@736229012 after retry RPC after error@1614031091) -> (continuation of Deferred@448501340 after retry RPC after error@1296206445) -> (continuation of Deferred@1773371399 after retry RPC after error@671178354), errback=(continuation of Deferred@1929086179 after retry RPC after error@1286887968) -> (continuation of Deferred@1979264459 after retry RPC after error@1752782059) -> (continuation of Deferred@1183521179 after retry RPC after error@679489072) -> (continuation of Deferred@346802590 after retry RPC after error@920062444) -> (continuation of Deferred@680429932 after retry RPC after error@22467555) -> (continuation of Deferred@1266355566 after retry RPC after error@1164830084) -> (continuation of Deferred@435470764 after retry RPC after error@780845242) -> (continuation of Deferred@1032435832 after retry RPC after error@400698248) -> (continuation of Deferred@681861497 after retry RPC after error@1356069831) -> (continuation of Deferred@543806429 after retry RPC after error@642843187) -> (continuation of Deferred@771035177 after retry RPC after error@1656212371) -> (continuation of Deferred@204348195 after retry RPC after error@574672911) -> (continuation of Deferred@657582134 after retry RPC after error@2072396816) -> (continuation of Deferred@126725952 after retry RPC after error@602523414) -> (continuation of Deferred@196213509 after retry RPC after error@1121880231) -> (continuation of Deferred@675534867 after retry RPC after error@1165982586) -> (continuation of Deferred@1091754243 after retry RPC after error@2000677430) -> (continuation of Deferred@1099573122 after retry RPC after error@2131763298) -> (continuation of Deferred@681391170 after retry RPC after error@544933181) -> (continuation of Deferred@819826823 after retry RPC after error@1311811022) -> (continuation of Deferred@265205614 after retry RPC after error@2011437383) -> (continuation of Deferred@736229012 after retry RPC after error@1614031091) -> (continuation of Deferred@448501340 after retry RPC after error@1296206445) -> (continuation of Deferred@1773371399 after retry RPC after error@671178354)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.4.97.254:35997
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=28171), Traces: [0ms] refreshing cache from master, [45ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.254:35997, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.252:45135, [123ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.253:46769, [175ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.254:35997: Network error: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [181ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.253:46769: Network error: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [518ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.252:45135: OK, [522ms] delaying RPC due to: Service unavailable: Master config (127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [536ms] refreshing cache from master, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.254:35997, [538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.252:45135, [541ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.254:35997: Network error: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [541ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.253:46769, [544ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.253:46769: Network error: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [547ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.252:45135: OK, [548ms] delaying RPC due to: Service unavailable: Master config (127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [555ms] refreshing cache from master, [555ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.254:35997, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.252:45135, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.253:46769, [561ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.253:46769: Network error: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [561ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.254:35997: Network error: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [563ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.252:45135: OK, [565ms] delaying RPC due to: Service unavailable: Master config (127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [575ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.254:35997, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.252:45135, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.253:46769, [580ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.253:46769: Network error: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [581ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.254:35997: Network error: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [583ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.252:45135: OK, [584ms] delaying RPC due to: Service unavailable: Master config (127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [595ms] refreshing cache from master, [595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.254:35997, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.252:45135, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.253:46769, [599ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.253:46769: Network error: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [601ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.254:35997: Network error: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [602ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.252:45135: OK, [603ms] delaying RPC due to: Service unavailable: Master config (127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [615ms] refreshing cache from master, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.254:35997, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.252:45135, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.253:46769, [620ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.253:46769: Network error: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [623ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.252:45135: OK, [624ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.254:35997: Network error: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [625ms] delaying RPC due to: Service unavailable: Master config (127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [675ms] refreshing cache from master, [675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.254:35997, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.252:45135, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.253:46769, [680ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.253:46769: Network error: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [681ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.254:35997: Network error: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [683ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.252:45135: OK, [684ms] delaying RPC due to: Service unavailable: Master config (127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [795ms] refreshing cache from master, [796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.254:35997, [797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.252:45135, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.253:46769, [800ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.253:46769: Network error: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [800ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.254:35997: Network error: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [802ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.252:45135: OK, [802ms] delaying RPC due to: Service unavailable: Master config (127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [836ms] refreshing cache from master, [836ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.254:35997, [837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.252:45135, [837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.253:46769, [840ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.253:46769: Network error: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [840ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.254:35997: Network error: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [841ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.252:45135: OK, [842ms] delaying RPC due to: Service unavailable: Master config (127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [855ms] refreshing cache from master, [855ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.254:35997, [857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.252:45135, [857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.253:46769, [859ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.253:46769: Network error: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [861ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.254:35997: Network error: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [862ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.252:45135: OK, [863ms] delaying RPC due to: Service unavailable: Master config (127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [1855ms] refreshing cache from master, [1856ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.254:35997, [1857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.252:45135, [1857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.253:46769, [1859ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.254:35997: Network error: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [1860ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.253:46769: Network error: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [1861ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.252:45135: OK, [1861ms] delaying RPC due to: Service unavailable: Master config (127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [2015ms] refreshing cache from master, [2015ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.254:35997, [2016ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.252:45135, [2022ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.253:46769, [2025ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.254:35997: Network error: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [2025ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.253:46769: Network error: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [2026ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.252:45135: OK, [2026ms] delaying RPC due to: Service unavailable: Master config (127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [3815ms] refreshing cache from master, [3815ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.254:35997, [3816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.252:45135, [3816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.253:46769, [3819ms] too many traces: truncated at 100 traces, deferred=Deferred@1871594101(state=PENDING, result=null, callback=(continuation of Deferred@1929086179 after retry RPC after error@1286887968) -> (continuation of Deferred@1979264459 after retry RPC after error@1752782059) -> (continuation of Deferred@1183521179 after retry RPC after error@679489072) -> (continuation of Deferred@346802590 after retry RPC after error@920062444) -> (continuation of Deferred@680429932 after retry RPC after error@22467555) -> (continuation of Deferred@1266355566 after retry RPC after error@1164830084) -> (continuation of Deferred@435470764 after retry RPC after error@780845242) -> (continuation of Deferred@1032435832 after retry RPC after error@400698248) -> (continuation of Deferred@681861497 after retry RPC after error@1356069831) -> (continuation of Deferred@543806429 after retry RPC after error@642843187) -> (continuation of Deferred@771035177 after retry RPC after error@1656212371) -> (continuation of Deferred@204348195 after retry RPC after error@574672911) -> (continuation of Deferred@657582134 after retry RPC after error@2072396816) -> (continuation of Deferred@126725952 after retry RPC after error@602523414) -> (continuation of Deferred@196213509 after retry RPC after error@1121880231) -> (continuation of Deferred@675534867 after retry RPC after error@1165982586) -> (continuation of Deferred@1091754243 after retry RPC after error@2000677430) -> (continuation of Deferred@1099573122 after retry RPC after error@2131763298) -> (continuation of Deferred@681391170 after retry RPC after error@544933181) -> (continuation of Deferred@819826823 after retry RPC after error@1311811022) -> (continuation of Deferred@265205614 after retry RPC after error@2011437383) -> (continuation of Deferred@736229012 after retry RPC after error@1614031091) -> (continuation of Deferred@448501340 after retry RPC after error@1296206445) -> (continuation of Deferred@1773371399 after retry RPC after error@671178354), errback=(continuation of Deferred@1929086179 after retry RPC after error@1286887968) -> (continuation of Deferred@1979264459 after retry RPC after error@1752782059) -> (continuation of Deferred@1183521179 after retry RPC after error@679489072) -> (continuation of Deferred@346802590 after retry RPC after error@920062444) -> (continuation of Deferred@680429932 after retry RPC after error@22467555) -> (continuation of Deferred@1266355566 after retry RPC after error@1164830084) -> (continuation of Deferred@435470764 after retry RPC after error@780845242) -> (continuation of Deferred@1032435832 after retry RPC after error@400698248) -> (continuation of Deferred@681861497 after retry RPC after error@1356069831) -> (continuation of Deferred@543806429 after retry RPC after error@642843187) -> (continuation of Deferred@771035177 after retry RPC after error@1656212371) -> (continuation of Deferred@204348195 after retry RPC after error@574672911) -> (continuation of Deferred@657582134 after retry RPC after error@2072396816) -> (continuation of Deferred@126725952 after retry RPC after error@602523414) -> (continuation of Deferred@196213509 after retry RPC after error@1121880231) -> (continuation of Deferred@675534867 after retry RPC after error@1165982586) -> (continuation of Deferred@1091754243 after retry RPC after error@2000677430) -> (continuation of Deferred@1099573122 after retry RPC after error@2131763298) -> (continuation of Deferred@681391170 after retry RPC after error@544933181) -> (continuation of Deferred@819826823 after retry RPC after error@1311811022) -> (continuation of Deferred@265205614 after retry RPC after error@2011437383) -> (continuation of Deferred@736229012 after retry RPC after error@1614031091) -> (continuation of Deferred@448501340 after retry RPC after error@1296206445) -> (continuation of Deferred@1773371399 after retry RPC after error@671178354)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.4.97.254:35997
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
03:47:05.311 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
03:47:05.311 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=28171), Traces: [0ms] refreshing cache from master, [45ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.254:35997, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.252:45135, [123ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.253:46769, [175ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.254:35997: Network error: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [181ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.253:46769: Network error: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [518ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.252:45135: OK, [522ms] delaying RPC due to: Service unavailable: Master config (127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [536ms] refreshing cache from master, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.254:35997, [538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.252:45135, [541ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.254:35997: Network error: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [541ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.253:46769, [544ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.253:46769: Network error: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [547ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.252:45135: OK, [548ms] delaying RPC due to: Service unavailable: Master config (127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [555ms] refreshing cache from master, [555ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.254:35997, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.252:45135, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.253:46769, [561ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.253:46769: Network error: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [561ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.254:35997: Network error: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [563ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.252:45135: OK, [565ms] delaying RPC due to: Service unavailable: Master config (127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [575ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.254:35997, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.252:45135, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.253:46769, [580ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.253:46769: Network error: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [581ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.254:35997: Network error: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [583ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.252:45135: OK, [584ms] delaying RPC due to: Service unavailable: Master config (127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [595ms] refreshing cache from master, [595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.254:35997, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.252:45135, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.253:46769, [599ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.253:46769: Network error: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [601ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.254:35997: Network error: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [602ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.252:45135: OK, [603ms] delaying RPC due to: Service unavailable: Master config (127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [615ms] refreshing cache from master, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.254:35997, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.252:45135, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.253:46769, [620ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.253:46769: Network error: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [623ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.252:45135: OK, [624ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.254:35997: Network error: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [625ms] delaying RPC due to: Service unavailable: Master config (127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [675ms] refreshing cache from master, [675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.254:35997, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.252:45135, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.253:46769, [680ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.253:46769: Network error: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [681ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.254:35997: Network error: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [683ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.252:45135: OK, [684ms] delaying RPC due to: Service unavailable: Master config (127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [795ms] refreshing cache from master, [796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.254:35997, [797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.252:45135, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.253:46769, [800ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.253:46769: Network error: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [800ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.254:35997: Network error: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [802ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.252:45135: OK, [802ms] delaying RPC due to: Service unavailable: Master config (127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [836ms] refreshing cache from master, [836ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.254:35997, [837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.252:45135, [837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.253:46769, [840ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.253:46769: Network error: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [840ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.254:35997: Network error: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [841ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.252:45135: OK, [842ms] delaying RPC due to: Service unavailable: Master config (127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [855ms] refreshing cache from master, [855ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.254:35997, [857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.252:45135, [857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.253:46769, [859ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.253:46769: Network error: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [861ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.254:35997: Network error: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [862ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.252:45135: OK, [863ms] delaying RPC due to: Service unavailable: Master config (127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [1855ms] refreshing cache from master, [1856ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.254:35997, [1857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.252:45135, [1857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.253:46769, [1859ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.254:35997: Network error: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [1860ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.253:46769: Network error: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [1861ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.252:45135: OK, [1861ms] delaying RPC due to: Service unavailable: Master config (127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [2015ms] refreshing cache from master, [2015ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.254:35997, [2016ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.252:45135, [2022ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.253:46769, [2025ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.254:35997: Network error: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [2025ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.253:46769: Network error: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [2026ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.252:45135: OK, [2026ms] delaying RPC due to: Service unavailable: Master config (127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [3815ms] refreshing cache from master, [3815ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.254:35997, [3816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.252:45135, [3816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.253:46769, [3819ms] too many traces: truncated at 100 traces, deferred=Deferred@1871594101(state=PENDING, result=null, callback=(continuation of Deferred@1929086179 after retry RPC after error@1286887968) -> (continuation of Deferred@1979264459 after retry RPC after error@1752782059) -> (continuation of Deferred@1183521179 after retry RPC after error@679489072) -> (continuation of Deferred@346802590 after retry RPC after error@920062444) -> (continuation of Deferred@680429932 after retry RPC after error@22467555) -> (continuation of Deferred@1266355566 after retry RPC after error@1164830084) -> (continuation of Deferred@435470764 after retry RPC after error@780845242) -> (continuation of Deferred@1032435832 after retry RPC after error@400698248) -> (continuation of Deferred@681861497 after retry RPC after error@1356069831) -> (continuation of Deferred@543806429 after retry RPC after error@642843187) -> (continuation of Deferred@771035177 after retry RPC after error@1656212371) -> (continuation of Deferred@204348195 after retry RPC after error@574672911) -> (continuation of Deferred@657582134 after retry RPC after error@2072396816) -> (continuation of Deferred@126725952 after retry RPC after error@602523414) -> (continuation of Deferred@196213509 after retry RPC after error@1121880231) -> (continuation of Deferred@675534867 after retry RPC after error@1165982586) -> (continuation of Deferred@1091754243 after retry RPC after error@2000677430) -> (continuation of Deferred@1099573122 after retry RPC after error@2131763298) -> (continuation of Deferred@681391170 after retry RPC after error@544933181) -> (continuation of Deferred@819826823 after retry RPC after error@1311811022) -> (continuation of Deferred@265205614 after retry RPC after error@2011437383) -> (continuation of Deferred@736229012 after retry RPC after error@1614031091) -> (continuation of Deferred@448501340 after retry RPC after error@1296206445) -> (continuation of Deferred@1773371399 after retry RPC after error@671178354), errback=(continuation of Deferred@1929086179 after retry RPC after error@1286887968) -> (continuation of Deferred@1979264459 after retry RPC after error@1752782059) -> (continuation of Deferred@1183521179 after retry RPC after error@679489072) -> (continuation of Deferred@346802590 after retry RPC after error@920062444) -> (continuation of Deferred@680429932 after retry RPC after error@22467555) -> (continuation of Deferred@1266355566 after retry RPC after error@1164830084) -> (continuation of Deferred@435470764 after retry RPC after error@780845242) -> (continuation of Deferred@1032435832 after retry RPC after error@400698248) -> (continuation of Deferred@681861497 after retry RPC after error@1356069831) -> (continuation of Deferred@543806429 after retry RPC after error@642843187) -> (continuation of Deferred@771035177 after retry RPC after error@1656212371) -> (continuation of Deferred@204348195 after retry RPC after error@574672911) -> (continuation of Deferred@657582134 after retry RPC after error@2072396816) -> (continuation of Deferred@126725952 after retry RPC after error@602523414) -> (continuation of Deferred@196213509 after retry RPC after error@1121880231) -> (continuation of Deferred@675534867 after retry RPC after error@1165982586) -> (continuation of Deferred@1091754243 after retry RPC after error@2000677430) -> (continuation of Deferred@1099573122 after retry RPC after error@2131763298) -> (continuation of Deferred@681391170 after retry RPC after error@544933181) -> (continuation of Deferred@819826823 after retry RPC after error@1311811022) -> (continuation of Deferred@265205614 after retry RPC after error@2011437383) -> (continuation of Deferred@736229012 after retry RPC after error@1614031091) -> (continuation of Deferred@448501340 after retry RPC after error@1296206445) -> (continuation of Deferred@1773371399 after retry RPC after error@671178354)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.4.97.254:35997
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=28171), Traces: [0ms] refreshing cache from master, [45ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.254:35997, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.252:45135, [123ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.253:46769, [175ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.254:35997: Network error: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [181ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.253:46769: Network error: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [518ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.252:45135: OK, [522ms] delaying RPC due to: Service unavailable: Master config (127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [536ms] refreshing cache from master, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.254:35997, [538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.252:45135, [541ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.254:35997: Network error: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [541ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.253:46769, [544ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.253:46769: Network error: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [547ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.252:45135: OK, [548ms] delaying RPC due to: Service unavailable: Master config (127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [555ms] refreshing cache from master, [555ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.254:35997, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.252:45135, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.253:46769, [561ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.253:46769: Network error: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [561ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.254:35997: Network error: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [563ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.252:45135: OK, [565ms] delaying RPC due to: Service unavailable: Master config (127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [575ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.254:35997, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.252:45135, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.253:46769, [580ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.253:46769: Network error: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [581ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.254:35997: Network error: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [583ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.252:45135: OK, [584ms] delaying RPC due to: Service unavailable: Master config (127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [595ms] refreshing cache from master, [595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.254:35997, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.252:45135, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.253:46769, [599ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.253:46769: Network error: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [601ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.254:35997: Network error: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [602ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.252:45135: OK, [603ms] delaying RPC due to: Service unavailable: Master config (127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [615ms] refreshing cache from master, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.254:35997, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.252:45135, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.253:46769, [620ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.253:46769: Network error: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [623ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.252:45135: OK, [624ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.254:35997: Network error: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [625ms] delaying RPC due to: Service unavailable: Master config (127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [675ms] refreshing cache from master, [675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.254:35997, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.252:45135, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.253:46769, [680ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.253:46769: Network error: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [681ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.254:35997: Network error: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [683ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.252:45135: OK, [684ms] delaying RPC due to: Service unavailable: Master config (127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [795ms] refreshing cache from master, [796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.254:35997, [797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.252:45135, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.253:46769, [800ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.253:46769: Network error: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [800ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.254:35997: Network error: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [802ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.252:45135: OK, [802ms] delaying RPC due to: Service unavailable: Master config (127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [836ms] refreshing cache from master, [836ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.254:35997, [837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.252:45135, [837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.253:46769, [840ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.253:46769: Network error: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [840ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.254:35997: Network error: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [841ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.252:45135: OK, [842ms] delaying RPC due to: Service unavailable: Master config (127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [855ms] refreshing cache from master, [855ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.254:35997, [857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.252:45135, [857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.253:46769, [859ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.253:46769: Network error: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [861ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.254:35997: Network error: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [862ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.252:45135: OK, [863ms] delaying RPC due to: Service unavailable: Master config (127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [1855ms] refreshing cache from master, [1856ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.254:35997, [1857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.252:45135, [1857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.253:46769, [1859ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.254:35997: Network error: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [1860ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.253:46769: Network error: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [1861ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.252:45135: OK, [1861ms] delaying RPC due to: Service unavailable: Master config (127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [2015ms] refreshing cache from master, [2015ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.254:35997, [2016ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.252:45135, [2022ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.253:46769, [2025ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.254:35997: Network error: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997, [2025ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.253:46769: Network error: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [2026ms] Sub RPC ConnectToMaster: received response from server master-127.4.97.252:45135: OK, [2026ms] delaying RPC due to: Service unavailable: Master config (127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769, [3815ms] refreshing cache from master, [3815ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.254:35997, [3816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.252:45135, [3816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.97.253:46769, [3819ms] too many traces: truncated at 100 traces, deferred=Deferred@1871594101(state=PENDING, result=null, callback=(continuation of Deferred@1929086179 after retry RPC after error@1286887968) -> (continuation of Deferred@1979264459 after retry RPC after error@1752782059) -> (continuation of Deferred@1183521179 after retry RPC after error@679489072) -> (continuation of Deferred@346802590 after retry RPC after error@920062444) -> (continuation of Deferred@680429932 after retry RPC after error@22467555) -> (continuation of Deferred@1266355566 after retry RPC after error@1164830084) -> (continuation of Deferred@435470764 after retry RPC after error@780845242) -> (continuation of Deferred@1032435832 after retry RPC after error@400698248) -> (continuation of Deferred@681861497 after retry RPC after error@1356069831) -> (continuation of Deferred@543806429 after retry RPC after error@642843187) -> (continuation of Deferred@771035177 after retry RPC after error@1656212371) -> (continuation of Deferred@204348195 after retry RPC after error@574672911) -> (continuation of Deferred@657582134 after retry RPC after error@2072396816) -> (continuation of Deferred@126725952 after retry RPC after error@602523414) -> (continuation of Deferred@196213509 after retry RPC after error@1121880231) -> (continuation of Deferred@675534867 after retry RPC after error@1165982586) -> (continuation of Deferred@1091754243 after retry RPC after error@2000677430) -> (continuation of Deferred@1099573122 after retry RPC after error@2131763298) -> (continuation of Deferred@681391170 after retry RPC after error@544933181) -> (continuation of Deferred@819826823 after retry RPC after error@1311811022) -> (continuation of Deferred@265205614 after retry RPC after error@2011437383) -> (continuation of Deferred@736229012 after retry RPC after error@1614031091) -> (continuation of Deferred@448501340 after retry RPC after error@1296206445) -> (continuation of Deferred@1773371399 after retry RPC after error@671178354), errback=(continuation of Deferred@1929086179 after retry RPC after error@1286887968) -> (continuation of Deferred@1979264459 after retry RPC after error@1752782059) -> (continuation of Deferred@1183521179 after retry RPC after error@679489072) -> (continuation of Deferred@346802590 after retry RPC after error@920062444) -> (continuation of Deferred@680429932 after retry RPC after error@22467555) -> (continuation of Deferred@1266355566 after retry RPC after error@1164830084) -> (continuation of Deferred@435470764 after retry RPC after error@780845242) -> (continuation of Deferred@1032435832 after retry RPC after error@400698248) -> (continuation of Deferred@681861497 after retry RPC after error@1356069831) -> (continuation of Deferred@543806429 after retry RPC after error@642843187) -> (continuation of Deferred@771035177 after retry RPC after error@1656212371) -> (continuation of Deferred@204348195 after retry RPC after error@574672911) -> (continuation of Deferred@657582134 after retry RPC after error@2072396816) -> (continuation of Deferred@126725952 after retry RPC after error@602523414) -> (continuation of Deferred@196213509 after retry RPC after error@1121880231) -> (continuation of Deferred@675534867 after retry RPC after error@1165982586) -> (continuation of Deferred@1091754243 after retry RPC after error@2000677430) -> (continuation of Deferred@1099573122 after retry RPC after error@2131763298) -> (continuation of Deferred@681391170 after retry RPC after error@544933181) -> (continuation of Deferred@819826823 after retry RPC after error@1311811022) -> (continuation of Deferred@265205614 after retry RPC after error@2011437383) -> (continuation of Deferred@736229012 after retry RPC after error@1614031091) -> (continuation of Deferred@448501340 after retry RPC after error@1296206445) -> (continuation of Deferred@1773371399 after retry RPC after error@671178354)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.4.97.254:35997,127.4.97.252:45135,127.4.97.253:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.253:46769(127.4.97.253:46769): Connection refused: /127.4.97.253:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.97.254:35997(127.4.97.254:35997): Connection refused: /127.4.97.254:35997
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.4.97.254:35997
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)