22:11:22.137 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
22:11:22.148 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskyO6VHJ/test-tmp
22:11:22.151 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
22:11:22.154 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskyO6VHJ/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
22:11:22.425 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
22:11:22.558 [DEBUG - main] (MiniKuduCluster.java:195) Response:
22:11:22.559 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
22:11:22.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:22.564189 7018 mini_oidc.cc:152] Starting JWKS server
22:11:22.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:22.569633 7018 webserver.cc:460] Webserver started at http://127.0.0.1:37839/ using document root <none> and password file <none>
22:11:22.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:22.570097 7018 mini_oidc.cc:189] Starting OIDC Discovery server
22:11:22.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:22.571990 7018 webserver.cc:460] Webserver started at http://127.0.0.1:43593/ using document root <none> and password file <none>
22:11:22.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T22:11:22Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
22:11:22.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T22:11:22Z Disabled control of system clock
22:11:22.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:22.619513 7018 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu
22:11:22.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu
22:11:22.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/master-0/wal
22:11:22.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/master-0/data
22:11:22.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
22:11:22.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
22:11:22.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
22:11:22.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:11:22.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:11:22.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:11:22.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/master-0/logs
22:11:22.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/master-0/data/info.pb
22:11:22.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:11:22.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
22:11:22.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
22:11:22.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
22:11:22.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
22:11:22.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:11:22.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:11:22.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:11:22.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
22:11:22.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
22:11:22.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
22:11:22.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
22:11:22.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.218.190:38037
22:11:22.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.218.190
22:11:22.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:11:22.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.218.148:45245
22:11:22.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:11:22.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:11:22.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:11:22.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
22:11:22.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.218.190:38037,127.6.218.189:41999,127.6.218.188:42947
22:11:22.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
22:11:22.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37839/jwks.json with env {}
22:11:22.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:22.905572 7030 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
22:11:22.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:22.906136 7030 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
22:11:22.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:22.906778 7030 flags.cc:425] Enabled unsafe flag: --never_fsync=true
22:11:22.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:22.936153 7030 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
22:11:22.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:22.936411 7030 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
22:11:22.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:22.936589 7030 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
22:11:22.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:22.936810 7030 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
22:11:22.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:22.937081 7030 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
22:11:22.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:22.937330 7030 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37839/jwks.json
22:11:22.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:22.971783 7030 master_runner.cc:386] Master server non-default flags:
22:11:22.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:11:22.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.218.148:45245
22:11:22.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:11:22.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:11:22.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/master-0/data
22:11:22.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/master-0/wal
22:11:22.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
22:11:22.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.218.190:38037,127.6.218.189:41999,127.6.218.188:42947
22:11:22.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:11:22.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:11:22.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
22:11:22.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.218.190:38037
22:11:22.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
22:11:22.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
22:11:22.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
22:11:22.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37839/jwks.json
22:11:22.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:11:22.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:11:22.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/master-0/data/info.pb
22:11:22.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.218.190
22:11:22.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:11:22.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
22:11:22.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:11:22.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
22:11:22.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
22:11:22.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:11:22.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/master-0/logs
22:11:22.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:11:22.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
22:11:22.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:11:22.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
22:11:22.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
22:11:22.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
22:11:22.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
22:11:22.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 21:22:24 UTC on ccycloud.quasar-wysbne.root.comops.site
22:11:22.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676318
22:11:22.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
22:11:22.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:22.973141 7030 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
22:11:22.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:22.975250 7030 file_cache.cc:493] Constructed file cache file cache with capacity 419430
22:11:22.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:22.995085 7036 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:11:24.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:22.995091 7037 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:11:24.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:22.995136 7039 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:11:24.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:22.995882 7030 server_base.cc:1034] running on GCE node
22:11:24.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:24.177482 7030 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
22:11:24.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:24.180727 7030 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
22:11:24.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:24.182160 7030 hybrid_clock.cc:648] HybridClock initialized: now 1790547084182116 us; error 64 us; skew 500 ppm
22:11:24.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:24.183007 7030 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
22:11:24.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:24.189664 7030 webserver.cc:460] Webserver started at http://127.6.218.190:41985/ using document root <none> and password file <none>
22:11:24.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:24.190593 7030 fs_manager.cc:359] Metadata directory not provided
22:11:24.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:24.190771 7030 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
22:11:24.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:24.191143 7030 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
22:11:24.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:24.195827 7030 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/master-0/data/instance:
22:11:24.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "44965596305b478e93a1a3c450737f25"
22:11:24.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 22:11:24 on dist-test-slave-qzth"
22:11:24.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:24.196951 7030 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/master-0/wal/instance:
22:11:24.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "44965596305b478e93a1a3c450737f25"
22:11:24.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 22:11:24 on dist-test-slave-qzth"
22:11:24.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:24.204484 7030 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
22:11:24.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:24.209327 7046 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
22:11:24.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:24.210352 7030 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
22:11:24.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:24.210618 7030 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/master-0/data,/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/master-0/wal
22:11:24.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "44965596305b478e93a1a3c450737f25"
22:11:24.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 22:11:24 on dist-test-slave-qzth"
22:11:24.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:24.210928 7030 fs_report.cc:383] FS layout report
22:11:24.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
22:11:24.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/master-0/wal
22:11:24.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/master-0/wal
22:11:24.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/master-0/data/data
22:11:24.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
22:11:24.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
22:11:24.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
22:11:24.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
22:11:24.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
22:11:24.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
22:11:24.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
22:11:24.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
22:11:24.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
22:11:24.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
22:11:24.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
22:11:24.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:24.248749 7030 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
22:11:24.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:24.250015 7030 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
22:11:24.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:24.250370 7030 kserver.cc:163] Server-wide thread pool size limit: 3276
22:11:24.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:24.322841 7030 rpc_server.cc:307] RPC server started. Bound to: 127.6.218.190:38037
22:11:24.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:24.322937 7097 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.218.190:38037 every 8 connection(s)
22:11:24.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:24.325284 7030 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/master-0/data/info.pb
22:11:24.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:24.326251 7018 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu as pid 7030
22:11:24.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:24.327230 7018 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/master-0/wal/instance
22:11:24.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:24.331835 7018 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu
22:11:24.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu
22:11:24.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/master-1/wal
22:11:24.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/master-1/data
22:11:24.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
22:11:24.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
22:11:24.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
22:11:24.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:11:24.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:11:24.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:11:24.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/master-1/logs
22:11:24.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/master-1/data/info.pb
22:11:24.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:11:24.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
22:11:24.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
22:11:24.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
22:11:24.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
22:11:24.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:11:24.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:11:24.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:11:24.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
22:11:24.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
22:11:24.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
22:11:24.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
22:11:24.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.218.189:41999
22:11:24.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.218.189
22:11:24.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:11:24.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.218.148:45245
22:11:24.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:11:24.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:11:24.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:11:24.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
22:11:24.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.218.190:38037,127.6.218.189:41999,127.6.218.188:42947
22:11:24.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
22:11:24.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37839/jwks.json with env {}
22:11:24.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:24.332000 7098 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
22:11:24.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:24.344225 7098 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.218.190" port: 38037 } has no permanent_uuid. Determining permanent_uuid...
22:11:24.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:24.372625 7098 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.218.189" port: 41999 } has no permanent_uuid. Determining permanent_uuid...
22:11:24.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:24.374980 7050 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.218.189:41999: connect: Connection refused (error 111)
22:11:24.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:24.378882 7098 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.218.189:41999: Network error: Client connection negotiation failed: client connection to 127.6.218.189:41999: connect: Connection refused (error 111)
22:11:24.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:24.428383 7098 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.218.189" port: 41999 } attempt: 1
22:11:24.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:24.431943 7098 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.218.189:41999: Network error: Client connection negotiation failed: client connection to 127.6.218.189:41999: connect: Connection refused (error 111)
22:11:24.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:24.500392 7098 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.218.189" port: 41999 } attempt: 2
22:11:24.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:24.504148 7098 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.218.189:41999: Network error: Client connection negotiation failed: client connection to 127.6.218.189:41999: connect: Connection refused (error 111)
22:11:24.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:24.595585 7098 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.218.189" port: 41999 } attempt: 3
22:11:24.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:24.599429 7098 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.218.189:41999: Network error: Client connection negotiation failed: client connection to 127.6.218.189:41999: connect: Connection refused (error 111)
22:11:24.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:24.640230 7100 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
22:11:24.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:24.640894 7100 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
22:11:24.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:24.641541 7100 flags.cc:425] Enabled unsafe flag: --never_fsync=true
22:11:24.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:24.670331 7100 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
22:11:24.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:24.670679 7100 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
22:11:24.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:24.671032 7100 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
22:11:24.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:24.671419 7100 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
22:11:24.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:24.671764 7100 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
22:11:24.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:24.672040 7100 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37839/jwks.json
22:11:24.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:24.704286 7100 master_runner.cc:386] Master server non-default flags:
22:11:24.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:11:24.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.218.148:45245
22:11:24.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:11:24.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:11:24.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/master-1/data
22:11:24.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/master-1/wal
22:11:24.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
22:11:24.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.218.190:38037,127.6.218.189:41999,127.6.218.188:42947
22:11:24.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:11:24.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:11:24.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
22:11:24.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.218.189:41999
22:11:24.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
22:11:24.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
22:11:24.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
22:11:24.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37839/jwks.json
22:11:24.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:11:24.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:11:24.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/master-1/data/info.pb
22:11:24.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.218.189
22:11:24.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:11:24.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
22:11:24.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:11:24.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
22:11:24.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
22:11:24.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:11:24.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/master-1/logs
22:11:24.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:11:24.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
22:11:24.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:11:24.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
22:11:24.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
22:11:24.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
22:11:24.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
22:11:24.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 21:22:24 UTC on ccycloud.quasar-wysbne.root.comops.site
22:11:24.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676318
22:11:24.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
22:11:24.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:24.705762 7100 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
22:11:24.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:24.707861 7100 file_cache.cc:493] Constructed file cache file cache with capacity 419430
22:11:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:24.727043 7109 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:11:24.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:24.742779 7098 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.218.189" port: 41999 } attempt: 4
22:11:24.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:24.746127 7098 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.218.189:41999: Network error: Client connection negotiation failed: client connection to 127.6.218.189:41999: connect: Connection refused (error 111)
22:11:25.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:25.045722 7098 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.218.189" port: 41999 } attempt: 5
22:11:25.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:25.049579 7098 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.218.189:41999: Network error: Client connection negotiation failed: client connection to 127.6.218.189:41999: connect: Connection refused (error 111)
22:11:25.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:25.597193 7098 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.218.189" port: 41999 } attempt: 6
22:11:25.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:25.604357 7098 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.218.189:41999: Network error: Client connection negotiation failed: client connection to 127.6.218.189:41999: connect: Connection refused (error 111)
22:11:25.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:24.727048 7112 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:11:25.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:24.728199 7100 server_base.cc:1034] running on GCE node
22:11:25.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:24.727061 7110 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:11:25.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:25.900468 7100 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
22:11:25.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:25.903083 7100 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
22:11:25.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:25.904445 7100 hybrid_clock.cc:648] HybridClock initialized: now 1790547085904406 us; error 55 us; skew 500 ppm
22:11:25.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:25.905555 7100 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
22:11:25.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:25.912806 7100 webserver.cc:460] Webserver started at http://127.6.218.189:34747/ using document root <none> and password file <none>
22:11:25.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:25.913873 7100 fs_manager.cc:359] Metadata directory not provided
22:11:25.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:25.914144 7100 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
22:11:25.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:25.914686 7100 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
22:11:25.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:25.919772 7100 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/master-1/data/instance:
22:11:25.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c4ceebdfcbbb41e3976917429d9ba525"
22:11:25.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 22:11:25 on dist-test-slave-qzth"
22:11:25.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:25.920966 7100 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/master-1/wal/instance:
22:11:25.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c4ceebdfcbbb41e3976917429d9ba525"
22:11:25.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 22:11:25 on dist-test-slave-qzth"
22:11:25.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:25.927766 7100 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.009s sys 0.000s
22:11:25.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:25.933223 7121 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
22:11:25.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:25.934296 7100 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
22:11:25.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:25.934564 7100 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/master-1/data,/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/master-1/wal
22:11:25.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c4ceebdfcbbb41e3976917429d9ba525"
22:11:25.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 22:11:25 on dist-test-slave-qzth"
22:11:25.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:25.934824 7100 fs_report.cc:383] FS layout report
22:11:25.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
22:11:25.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/master-1/wal
22:11:25.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/master-1/wal
22:11:25.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/master-1/data/data
22:11:25.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
22:11:25.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
22:11:25.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
22:11:25.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
22:11:25.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
22:11:25.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
22:11:25.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
22:11:25.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
22:11:25.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
22:11:25.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
22:11:25.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
22:11:25.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:25.975644 7100 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
22:11:25.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:25.976998 7100 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
22:11:25.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:25.977388 7100 kserver.cc:163] Server-wide thread pool size limit: 3276
22:11:26.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:26.051349 7100 rpc_server.cc:307] RPC server started. Bound to: 127.6.218.189:41999
22:11:26.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:26.051441 7172 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.218.189:41999 every 8 connection(s)
22:11:26.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:26.054251 7100 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/master-1/data/info.pb
22:11:26.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:26.059512 7173 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
22:11:26.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:26.060882 7018 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu as pid 7100
22:11:26.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:26.061305 7018 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/master-1/wal/instance
22:11:26.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:26.063366 7018 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu
22:11:26.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu
22:11:26.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/master-2/wal
22:11:26.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/master-2/data
22:11:26.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
22:11:26.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
22:11:26.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
22:11:26.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:11:26.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:11:26.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:11:26.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/master-2/logs
22:11:26.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/master-2/data/info.pb
22:11:26.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:11:26.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
22:11:26.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
22:11:26.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
22:11:26.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
22:11:26.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:11:26.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:11:26.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:11:26.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
22:11:26.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
22:11:26.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
22:11:26.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
22:11:26.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.218.188:42947
22:11:26.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.218.188
22:11:26.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:11:26.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.218.148:45245
22:11:26.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:11:26.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:11:26.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:11:26.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
22:11:26.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.218.190:38037,127.6.218.189:41999,127.6.218.188:42947
22:11:26.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
22:11:26.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37839/jwks.json with env {}
22:11:26.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:26.072507 7173 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.218.190" port: 38037 } has no permanent_uuid. Determining permanent_uuid...
22:11:26.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:26.095633 7173 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.218.189" port: 41999 } has no permanent_uuid. Determining permanent_uuid...
22:11:26.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:26.111706 7173 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.218.188" port: 42947 } has no permanent_uuid. Determining permanent_uuid...
22:11:26.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:26.113623 7123 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.218.188:42947: connect: Connection refused (error 111)
22:11:26.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:26.117726 7173 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.218.188:42947: Network error: Client connection negotiation failed: client connection to 127.6.218.188:42947: connect: Connection refused (error 111)
22:11:26.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:26.167296 7173 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.218.188" port: 42947 } attempt: 1
22:11:26.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:26.170938 7173 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.218.188:42947: Network error: Client connection negotiation failed: client connection to 127.6.218.188:42947: connect: Connection refused (error 111)
22:11:26.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:26.239405 7173 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.218.188" port: 42947 } attempt: 2
22:11:26.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:26.243314 7173 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.218.188:42947: Network error: Client connection negotiation failed: client connection to 127.6.218.188:42947: connect: Connection refused (error 111)
22:11:26.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:26.334712 7173 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.218.188" port: 42947 } attempt: 3
22:11:26.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:26.338286 7173 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.218.188:42947: Network error: Client connection negotiation failed: client connection to 127.6.218.188:42947: connect: Connection refused (error 111)
22:11:26.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:26.378600 7175 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
22:11:26.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:26.379211 7175 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
22:11:26.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:26.379670 7175 flags.cc:425] Enabled unsafe flag: --never_fsync=true
22:11:26.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:26.408010 7175 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
22:11:26.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:26.408241 7175 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
22:11:26.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:26.408418 7175 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
22:11:26.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:26.408597 7175 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
22:11:26.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:26.408859 7175 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
22:11:26.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:26.408998 7175 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37839/jwks.json
22:11:26.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:26.441020 7175 master_runner.cc:386] Master server non-default flags:
22:11:26.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:11:26.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.218.148:45245
22:11:26.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:11:26.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:11:26.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/master-2/data
22:11:26.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/master-2/wal
22:11:26.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
22:11:26.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.218.190:38037,127.6.218.189:41999,127.6.218.188:42947
22:11:26.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:11:26.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:11:26.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
22:11:26.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.218.188:42947
22:11:26.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
22:11:26.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
22:11:26.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
22:11:26.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37839/jwks.json
22:11:26.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:11:26.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:11:26.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/master-2/data/info.pb
22:11:26.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.218.188
22:11:26.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:11:26.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
22:11:26.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:11:26.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
22:11:26.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
22:11:26.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:11:26.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/master-2/logs
22:11:26.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:11:26.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
22:11:26.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:11:26.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
22:11:26.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
22:11:26.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
22:11:26.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
22:11:26.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 21:22:24 UTC on ccycloud.quasar-wysbne.root.comops.site
22:11:26.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676318
22:11:26.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
22:11:26.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:26.442237 7175 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
22:11:26.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:26.444193 7175 file_cache.cc:493] Constructed file cache file cache with capacity 419430
22:11:26.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:26.463837 7186 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:11:26.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:26.481699 7173 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.218.188" port: 42947 } attempt: 4
22:11:26.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:26.485498 7173 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.218.188:42947: Network error: Client connection negotiation failed: client connection to 127.6.218.188:42947: connect: Connection refused (error 111)
22:11:26.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:26.664939 7098 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.218.189" port: 41999 } attempt: 7
22:11:26.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:26.691589 7098 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.218.188" port: 42947 } has no permanent_uuid. Determining permanent_uuid...
22:11:26.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:26.696820 7098 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.218.188:42947: Network error: Client connection negotiation failed: client connection to 127.6.218.188:42947: connect: Connection refused (error 111)
22:11:26.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:26.755380 7098 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.218.188" port: 42947 } attempt: 1
22:11:26.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:26.759549 7098 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.218.188:42947: Network error: Client connection negotiation failed: client connection to 127.6.218.188:42947: connect: Connection refused (error 111)
22:11:26.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:26.785068 7173 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.218.188" port: 42947 } attempt: 5
22:11:26.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:26.788641 7173 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.218.188:42947: Network error: Client connection negotiation failed: client connection to 127.6.218.188:42947: connect: Connection refused (error 111)
22:11:26.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:26.841118 7098 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.218.188" port: 42947 } attempt: 2
22:11:26.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:26.845777 7098 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.218.188:42947: Network error: Client connection negotiation failed: client connection to 127.6.218.188:42947: connect: Connection refused (error 111)
22:11:26.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:26.931186 7098 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.218.188" port: 42947 } attempt: 3
22:11:26.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:26.935283 7098 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.218.188:42947: Network error: Client connection negotiation failed: client connection to 127.6.218.188:42947: connect: Connection refused (error 111)
22:11:27.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:27.075652 7098 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.218.188" port: 42947 } attempt: 4
22:11:27.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:27.080113 7098 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.218.188:42947: Network error: Client connection negotiation failed: client connection to 127.6.218.188:42947: connect: Connection refused (error 111)
22:11:27.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:27.336203 7173 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.218.188" port: 42947 } attempt: 6
22:11:27.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:27.343415 7173 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.218.188:42947: Network error: Client connection negotiation failed: client connection to 127.6.218.188:42947: connect: Connection refused (error 111)
22:11:27.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:27.363482 7098 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.218.188" port: 42947 } attempt: 5
22:11:27.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:27.367555 7098 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.218.188:42947: Network error: Client connection negotiation failed: client connection to 127.6.218.188:42947: connect: Connection refused (error 111)
22:11:27.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:26.464584 7188 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:11:27.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:26.464581 7185 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:11:27.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:26.465927 7175 server_base.cc:1034] running on GCE node
22:11:27.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:27.653405 7175 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
22:11:27.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:27.656594 7175 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
22:11:27.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:27.657940 7175 hybrid_clock.cc:648] HybridClock initialized: now 1790547087657911 us; error 41 us; skew 500 ppm
22:11:27.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:27.658798 7175 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
22:11:27.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:27.665886 7175 webserver.cc:460] Webserver started at http://127.6.218.188:44119/ using document root <none> and password file <none>
22:11:27.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:27.666841 7175 fs_manager.cc:359] Metadata directory not provided
22:11:27.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:27.667073 7175 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
22:11:27.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:27.667683 7175 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
22:11:27.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:27.672469 7175 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/master-2/data/instance:
22:11:27.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "642848d3f5d14c9fa14e917104636c26"
22:11:27.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 22:11:27 on dist-test-slave-qzth"
22:11:27.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:27.673595 7175 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/master-2/wal/instance:
22:11:27.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "642848d3f5d14c9fa14e917104636c26"
22:11:27.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 22:11:27 on dist-test-slave-qzth"
22:11:27.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:27.680768 7175 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.001s sys 0.004s
22:11:27.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:27.686374 7200 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
22:11:27.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:27.687278 7175 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
22:11:27.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:27.687606 7175 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/master-2/data,/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/master-2/wal
22:11:27.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "642848d3f5d14c9fa14e917104636c26"
22:11:27.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 22:11:27 on dist-test-slave-qzth"
22:11:27.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:27.687981 7175 fs_report.cc:383] FS layout report
22:11:27.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
22:11:27.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/master-2/wal
22:11:27.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/master-2/wal
22:11:27.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/master-2/data/data
22:11:27.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
22:11:27.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
22:11:27.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
22:11:27.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
22:11:27.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
22:11:27.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
22:11:27.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
22:11:27.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
22:11:27.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
22:11:27.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
22:11:27.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
22:11:27.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:27.720295 7175 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
22:11:27.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:27.721765 7175 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
22:11:27.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:27.722700 7175 kserver.cc:163] Server-wide thread pool size limit: 3276
22:11:27.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:27.795068 7175 rpc_server.cc:307] RPC server started. Bound to: 127.6.218.188:42947
22:11:27.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:27.795161 7251 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.218.188:42947 every 8 connection(s)
22:11:27.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:27.797737 7175 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/master-2/data/info.pb
22:11:27.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:27.800606 7018 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu as pid 7175
22:11:27.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:27.801167 7018 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/master-2/wal/instance
22:11:27.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:27.803412 7252 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
22:11:27.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:27.821364 7252 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.218.190" port: 38037 } has no permanent_uuid. Determining permanent_uuid...
22:11:27.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:27.848455 7252 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.218.189" port: 41999 } has no permanent_uuid. Determining permanent_uuid...
22:11:27.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:27.861290 7252 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.218.188" port: 42947 } has no permanent_uuid. Determining permanent_uuid...
22:11:27.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:27.878288 7252 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 642848d3f5d14c9fa14e917104636c26: Bootstrap starting.
22:11:27.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:27.884341 7252 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 642848d3f5d14c9fa14e917104636c26: Neither blocks nor log segments found. Creating new log.
22:11:27.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:27.886348 7252 log.cc:824] T 00000000000000000000000000000000 P 642848d3f5d14c9fa14e917104636c26: Log is configured to *not* fsync() on all Append() calls
22:11:27.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:27.891034 7252 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 642848d3f5d14c9fa14e917104636c26: No bootstrap required, opened a new log
22:11:27.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:27.911055 7252 raft_consensus.cc:348] T 00000000000000000000000000000000 P 642848d3f5d14c9fa14e917104636c26 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "44965596305b478e93a1a3c450737f25" member_type: VOTER last_known_addr { host: "127.6.218.190" port: 38037 } } peers { permanent_uuid: "c4ceebdfcbbb41e3976917429d9ba525" member_type: VOTER last_known_addr { host: "127.6.218.189" port: 41999 } } peers { permanent_uuid: "642848d3f5d14c9fa14e917104636c26" member_type: VOTER last_known_addr { host: "127.6.218.188" port: 42947 } }
22:11:27.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:27.911662 7252 raft_consensus.cc:374] T 00000000000000000000000000000000 P 642848d3f5d14c9fa14e917104636c26 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
22:11:27.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:27.911909 7252 raft_consensus.cc:729] T 00000000000000000000000000000000 P 642848d3f5d14c9fa14e917104636c26 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 642848d3f5d14c9fa14e917104636c26, State: Initialized, Role: FOLLOWER
22:11:27.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:27.912665 7252 consensus_queue.cc:260] T 00000000000000000000000000000000 P 642848d3f5d14c9fa14e917104636c26 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "44965596305b478e93a1a3c450737f25" member_type: VOTER last_known_addr { host: "127.6.218.190" port: 38037 } } peers { permanent_uuid: "c4ceebdfcbbb41e3976917429d9ba525" member_type: VOTER last_known_addr { host: "127.6.218.189" port: 41999 } } peers { permanent_uuid: "642848d3f5d14c9fa14e917104636c26" member_type: VOTER last_known_addr { host: "127.6.218.188" port: 42947 } }
22:11:27.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:27.915539 7261 sys_catalog.cc:455] T 00000000000000000000000000000000 P 642848d3f5d14c9fa14e917104636c26 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "44965596305b478e93a1a3c450737f25" member_type: VOTER last_known_addr { host: "127.6.218.190" port: 38037 } } peers { permanent_uuid: "c4ceebdfcbbb41e3976917429d9ba525" member_type: VOTER last_known_addr { host: "127.6.218.189" port: 41999 } } peers { permanent_uuid: "642848d3f5d14c9fa14e917104636c26" member_type: VOTER last_known_addr { host: "127.6.218.188" port: 42947 } } }
22:11:27.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:27.916337 7261 sys_catalog.cc:458] T 00000000000000000000000000000000 P 642848d3f5d14c9fa14e917104636c26 [sys.catalog]: This master's current role is: FOLLOWER
22:11:27.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:27.916821 7252 sys_catalog.cc:564] T 00000000000000000000000000000000 P 642848d3f5d14c9fa14e917104636c26 [sys.catalog]: configured and running, proceeding with master startup.
22:11:27.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:27.919943 7098 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.218.188" port: 42947 } attempt: 6
22:11:27.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:27.949494 7098 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 44965596305b478e93a1a3c450737f25: Bootstrap starting.
22:11:27.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:27.952993 7273 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 642848d3f5d14c9fa14e917104636c26: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:11:27.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:27.953310 7273 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:11:27.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:27.956907 7098 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 44965596305b478e93a1a3c450737f25: Neither blocks nor log segments found. Creating new log.
22:11:27.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:27.959352 7098 log.cc:824] T 00000000000000000000000000000000 P 44965596305b478e93a1a3c450737f25: Log is configured to *not* fsync() on all Append() calls
22:11:27.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:27.969051 7098 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 44965596305b478e93a1a3c450737f25: No bootstrap required, opened a new log
22:11:27.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:27.977562 7206 tablet.cc:2369] T 00000000000000000000000000000000 P 642848d3f5d14c9fa14e917104636c26: Can't schedule compaction. Clean time has not been advanced past its initial value.
22:11:27.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:27.979009 7261 raft_consensus.cc:482] T 00000000000000000000000000000000 P 642848d3f5d14c9fa14e917104636c26 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
22:11:27.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:27.979440 7261 raft_consensus.cc:504] T 00000000000000000000000000000000 P 642848d3f5d14c9fa14e917104636c26 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "44965596305b478e93a1a3c450737f25" member_type: VOTER last_known_addr { host: "127.6.218.190" port: 38037 } } peers { permanent_uuid: "c4ceebdfcbbb41e3976917429d9ba525" member_type: VOTER last_known_addr { host: "127.6.218.189" port: 41999 } } peers { permanent_uuid: "642848d3f5d14c9fa14e917104636c26" member_type: VOTER last_known_addr { host: "127.6.218.188" port: 42947 } }
22:11:27.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:27.982472 7261 leader_election.cc:288] T 00000000000000000000000000000000 P 642848d3f5d14c9fa14e917104636c26 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 44965596305b478e93a1a3c450737f25 (127.6.218.190:38037), c4ceebdfcbbb41e3976917429d9ba525 (127.6.218.189:41999)
22:11:27.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:27.947757 7175 master_runner.cc:186] Error getting master registration for 127.6.218.190:38037: OK, instance_id { permanent_uuid: "44965596305b478e93a1a3c450737f25" instance_seqno: 1790547084303083 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
22:11:27.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:27.985661 7175 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
22:11:28.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.002018 7098 raft_consensus.cc:348] T 00000000000000000000000000000000 P 44965596305b478e93a1a3c450737f25 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "44965596305b478e93a1a3c450737f25" member_type: VOTER last_known_addr { host: "127.6.218.190" port: 38037 } } peers { permanent_uuid: "c4ceebdfcbbb41e3976917429d9ba525" member_type: VOTER last_known_addr { host: "127.6.218.189" port: 41999 } } peers { permanent_uuid: "642848d3f5d14c9fa14e917104636c26" member_type: VOTER last_known_addr { host: "127.6.218.188" port: 42947 } }
22:11:28.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.003113 7098 raft_consensus.cc:374] T 00000000000000000000000000000000 P 44965596305b478e93a1a3c450737f25 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
22:11:28.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.003409 7098 raft_consensus.cc:729] T 00000000000000000000000000000000 P 44965596305b478e93a1a3c450737f25 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 44965596305b478e93a1a3c450737f25, State: Initialized, Role: FOLLOWER
22:11:28.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.004280 7098 consensus_queue.cc:260] T 00000000000000000000000000000000 P 44965596305b478e93a1a3c450737f25 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "44965596305b478e93a1a3c450737f25" member_type: VOTER last_known_addr { host: "127.6.218.190" port: 38037 } } peers { permanent_uuid: "c4ceebdfcbbb41e3976917429d9ba525" member_type: VOTER last_known_addr { host: "127.6.218.189" port: 41999 } } peers { permanent_uuid: "642848d3f5d14c9fa14e917104636c26" member_type: VOTER last_known_addr { host: "127.6.218.188" port: 42947 } }
22:11:28.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.007846 7276 sys_catalog.cc:455] T 00000000000000000000000000000000 P 44965596305b478e93a1a3c450737f25 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "44965596305b478e93a1a3c450737f25" member_type: VOTER last_known_addr { host: "127.6.218.190" port: 38037 } } peers { permanent_uuid: "c4ceebdfcbbb41e3976917429d9ba525" member_type: VOTER last_known_addr { host: "127.6.218.189" port: 41999 } } peers { permanent_uuid: "642848d3f5d14c9fa14e917104636c26" member_type: VOTER last_known_addr { host: "127.6.218.188" port: 42947 } } }
22:11:28.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.008890 7276 sys_catalog.cc:458] T 00000000000000000000000000000000 P 44965596305b478e93a1a3c450737f25 [sys.catalog]: This master's current role is: FOLLOWER
22:11:28.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.009545 7098 sys_catalog.cc:564] T 00000000000000000000000000000000 P 44965596305b478e93a1a3c450737f25 [sys.catalog]: configured and running, proceeding with master startup.
22:11:28.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.009740 7073 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "642848d3f5d14c9fa14e917104636c26" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "44965596305b478e93a1a3c450737f25" is_pre_election: true
22:11:28.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.010594 7073 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 44965596305b478e93a1a3c450737f25 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 642848d3f5d14c9fa14e917104636c26 in term 0.
22:11:28.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.012187 7202 leader_election.cc:302] T 00000000000000000000000000000000 P 642848d3f5d14c9fa14e917104636c26 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 44965596305b478e93a1a3c450737f25, 642848d3f5d14c9fa14e917104636c26; no voters:
22:11:28.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.013115 7261 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 642848d3f5d14c9fa14e917104636c26 [term 0 FOLLOWER]: Leader pre-election won for term 1
22:11:28.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.013440 7261 raft_consensus.cc:482] T 00000000000000000000000000000000 P 642848d3f5d14c9fa14e917104636c26 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
22:11:28.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.013720 7261 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 642848d3f5d14c9fa14e917104636c26 [term 0 FOLLOWER]: Advancing to term 1
22:11:28.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.020377 7261 raft_consensus.cc:504] T 00000000000000000000000000000000 P 642848d3f5d14c9fa14e917104636c26 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "44965596305b478e93a1a3c450737f25" member_type: VOTER last_known_addr { host: "127.6.218.190" port: 38037 } } peers { permanent_uuid: "c4ceebdfcbbb41e3976917429d9ba525" member_type: VOTER last_known_addr { host: "127.6.218.189" port: 41999 } } peers { permanent_uuid: "642848d3f5d14c9fa14e917104636c26" member_type: VOTER last_known_addr { host: "127.6.218.188" port: 42947 } }
22:11:28.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.028790 7261 leader_election.cc:288] T 00000000000000000000000000000000 P 642848d3f5d14c9fa14e917104636c26 [CANDIDATE]: Term 1 election: Requested vote from peers 44965596305b478e93a1a3c450737f25 (127.6.218.190:38037), c4ceebdfcbbb41e3976917429d9ba525 (127.6.218.189:41999)
22:11:28.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.030869 7073 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "642848d3f5d14c9fa14e917104636c26" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "44965596305b478e93a1a3c450737f25"
22:11:28.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.031423 7073 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 44965596305b478e93a1a3c450737f25 [term 0 FOLLOWER]: Advancing to term 1
22:11:28.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.038554 7073 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 44965596305b478e93a1a3c450737f25 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 642848d3f5d14c9fa14e917104636c26 in term 1.
22:11:28.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.039687 7202 leader_election.cc:302] T 00000000000000000000000000000000 P 642848d3f5d14c9fa14e917104636c26 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 44965596305b478e93a1a3c450737f25, 642848d3f5d14c9fa14e917104636c26; no voters:
22:11:28.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.040594 7261 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 642848d3f5d14c9fa14e917104636c26 [term 1 FOLLOWER]: Leader election won for term 1
22:11:28.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.043797 7261 raft_consensus.cc:686] T 00000000000000000000000000000000 P 642848d3f5d14c9fa14e917104636c26 [term 1 LEADER]: Becoming Leader. State: Replica: 642848d3f5d14c9fa14e917104636c26, State: Running, Role: LEADER
22:11:28.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.044895 7261 consensus_queue.cc:237] T 00000000000000000000000000000000 P 642848d3f5d14c9fa14e917104636c26 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "44965596305b478e93a1a3c450737f25" member_type: VOTER last_known_addr { host: "127.6.218.190" port: 38037 } } peers { permanent_uuid: "c4ceebdfcbbb41e3976917429d9ba525" member_type: VOTER last_known_addr { host: "127.6.218.189" port: 41999 } } peers { permanent_uuid: "642848d3f5d14c9fa14e917104636c26" member_type: VOTER last_known_addr { host: "127.6.218.188" port: 42947 } }
22:11:28.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.063817 7282 sys_catalog.cc:455] T 00000000000000000000000000000000 P 642848d3f5d14c9fa14e917104636c26 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 642848d3f5d14c9fa14e917104636c26. Latest consensus state: current_term: 1 leader_uuid: "642848d3f5d14c9fa14e917104636c26" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "44965596305b478e93a1a3c450737f25" member_type: VOTER last_known_addr { host: "127.6.218.190" port: 38037 } } peers { permanent_uuid: "c4ceebdfcbbb41e3976917429d9ba525" member_type: VOTER last_known_addr { host: "127.6.218.189" port: 41999 } } peers { permanent_uuid: "642848d3f5d14c9fa14e917104636c26" member_type: VOTER last_known_addr { host: "127.6.218.188" port: 42947 } } }
22:11:28.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.067049 7282 sys_catalog.cc:458] T 00000000000000000000000000000000 P 642848d3f5d14c9fa14e917104636c26 [sys.catalog]: This master's current role is: LEADER
22:11:28.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.079422 7287 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
22:11:28.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.095460 7287 catalog_manager.cc:1484] Initializing Kudu cluster ID...
22:11:28.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.129397 7073 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 44965596305b478e93a1a3c450737f25 [term 1 FOLLOWER]: Refusing update from remote peer 642848d3f5d14c9fa14e917104636c26: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
22:11:28.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.132259 7282 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 642848d3f5d14c9fa14e917104636c26 [LEADER]: Connected to new peer: Peer: permanent_uuid: "44965596305b478e93a1a3c450737f25" member_type: VOTER last_known_addr { host: "127.6.218.190" port: 38037 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
22:11:28.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.163960 7276 sys_catalog.cc:455] T 00000000000000000000000000000000 P 44965596305b478e93a1a3c450737f25 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 642848d3f5d14c9fa14e917104636c26. Latest consensus state: current_term: 1 leader_uuid: "642848d3f5d14c9fa14e917104636c26" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "44965596305b478e93a1a3c450737f25" member_type: VOTER last_known_addr { host: "127.6.218.190" port: 38037 } } peers { permanent_uuid: "c4ceebdfcbbb41e3976917429d9ba525" member_type: VOTER last_known_addr { host: "127.6.218.189" port: 41999 } } peers { permanent_uuid: "642848d3f5d14c9fa14e917104636c26" member_type: VOTER last_known_addr { host: "127.6.218.188" port: 42947 } } }
22:11:28.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.164680 7276 sys_catalog.cc:458] T 00000000000000000000000000000000 P 44965596305b478e93a1a3c450737f25 [sys.catalog]: This master's current role is: FOLLOWER
22:11:28.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.178468 7292 mvcc.cc:205] Tried to move back new op lower bound from 7334080872948654080 to 7334080872645005312. Current Snapshot: MvccSnapshot[applied={T|T < 7334080872948654080}]
22:11:28.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.182866 7261 sys_catalog.cc:455] T 00000000000000000000000000000000 P 642848d3f5d14c9fa14e917104636c26 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "642848d3f5d14c9fa14e917104636c26" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "44965596305b478e93a1a3c450737f25" member_type: VOTER last_known_addr { host: "127.6.218.190" port: 38037 } } peers { permanent_uuid: "c4ceebdfcbbb41e3976917429d9ba525" member_type: VOTER last_known_addr { host: "127.6.218.189" port: 41999 } } peers { permanent_uuid: "642848d3f5d14c9fa14e917104636c26" member_type: VOTER last_known_addr { host: "127.6.218.188" port: 42947 } } }
22:11:28.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.183771 7261 sys_catalog.cc:458] T 00000000000000000000000000000000 P 642848d3f5d14c9fa14e917104636c26 [sys.catalog]: This master's current role is: LEADER
22:11:28.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.185333 7287 catalog_manager.cc:1347] Generated new cluster ID: c2acd939602248d3834e64481eb92d3b
22:11:28.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.185588 7287 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
22:11:28.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.203871 7276 sys_catalog.cc:455] T 00000000000000000000000000000000 P 44965596305b478e93a1a3c450737f25 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "642848d3f5d14c9fa14e917104636c26" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "44965596305b478e93a1a3c450737f25" member_type: VOTER last_known_addr { host: "127.6.218.190" port: 38037 } } peers { permanent_uuid: "c4ceebdfcbbb41e3976917429d9ba525" member_type: VOTER last_known_addr { host: "127.6.218.189" port: 41999 } } peers { permanent_uuid: "642848d3f5d14c9fa14e917104636c26" member_type: VOTER last_known_addr { host: "127.6.218.188" port: 42947 } } }
22:11:28.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.205022 7276 sys_catalog.cc:458] T 00000000000000000000000000000000 P 44965596305b478e93a1a3c450737f25 [sys.catalog]: This master's current role is: FOLLOWER
22:11:28.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.171209 7030 master_runner.cc:186] Error getting master registration for 127.6.218.189:41999: OK, instance_id { permanent_uuid: "c4ceebdfcbbb41e3976917429d9ba525" instance_seqno: 1790547086031444 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
22:11:28.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.217509 7030 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
22:11:28.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:28.217113 7290 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 44965596305b478e93a1a3c450737f25: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:11:28.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:28.218187 7290 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:11:28.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.231206 7287 catalog_manager.cc:1370] Generated new certificate authority record
22:11:28.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.233879 7287 catalog_manager.cc:1504] Loading token signing keys...
22:11:28.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.270057 7287 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 642848d3f5d14c9fa14e917104636c26: Generated new TSK 0
22:11:28.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.272003 7287 catalog_manager.cc:1514] Initializing in-progress tserver states...
22:11:28.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.285100 7018 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu
22:11:28.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu
22:11:28.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/ts-0/wal
22:11:28.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/ts-0/data
22:11:28.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
22:11:28.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
22:11:28.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
22:11:28.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:11:28.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:11:28.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:11:28.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/ts-0/logs
22:11:28.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/ts-0/data/info.pb
22:11:28.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:11:28.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
22:11:28.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
22:11:28.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
22:11:28.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
22:11:28.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:11:28.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:11:28.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:11:28.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
22:11:28.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
22:11:28.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.218.129:0
22:11:28.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.218.129
22:11:28.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.218.129
22:11:28.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:11:28.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.218.190:38037,127.6.218.189:41999,127.6.218.188:42947
22:11:28.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.218.148:45245
22:11:28.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:11:28.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:11:28.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
22:11:28.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.403903 7173 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.218.188" port: 42947 } attempt: 7
22:11:28.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.466460 7173 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c4ceebdfcbbb41e3976917429d9ba525: Bootstrap starting.
22:11:28.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.485672 7173 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P c4ceebdfcbbb41e3976917429d9ba525: Neither blocks nor log segments found. Creating new log.
22:11:28.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.491600 7173 log.cc:824] T 00000000000000000000000000000000 P c4ceebdfcbbb41e3976917429d9ba525: Log is configured to *not* fsync() on all Append() calls
22:11:28.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.515206 7173 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c4ceebdfcbbb41e3976917429d9ba525: No bootstrap required, opened a new log
22:11:28.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.605697 7173 raft_consensus.cc:348] T 00000000000000000000000000000000 P c4ceebdfcbbb41e3976917429d9ba525 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "44965596305b478e93a1a3c450737f25" member_type: VOTER last_known_addr { host: "127.6.218.190" port: 38037 } } peers { permanent_uuid: "c4ceebdfcbbb41e3976917429d9ba525" member_type: VOTER last_known_addr { host: "127.6.218.189" port: 41999 } } peers { permanent_uuid: "642848d3f5d14c9fa14e917104636c26" member_type: VOTER last_known_addr { host: "127.6.218.188" port: 42947 } }
22:11:28.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.608497 7173 raft_consensus.cc:374] T 00000000000000000000000000000000 P c4ceebdfcbbb41e3976917429d9ba525 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
22:11:28.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.609431 7173 raft_consensus.cc:729] T 00000000000000000000000000000000 P c4ceebdfcbbb41e3976917429d9ba525 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c4ceebdfcbbb41e3976917429d9ba525, State: Initialized, Role: FOLLOWER
22:11:28.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.611966 7173 consensus_queue.cc:260] T 00000000000000000000000000000000 P c4ceebdfcbbb41e3976917429d9ba525 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "44965596305b478e93a1a3c450737f25" member_type: VOTER last_known_addr { host: "127.6.218.190" port: 38037 } } peers { permanent_uuid: "c4ceebdfcbbb41e3976917429d9ba525" member_type: VOTER last_known_addr { host: "127.6.218.189" port: 41999 } } peers { permanent_uuid: "642848d3f5d14c9fa14e917104636c26" member_type: VOTER last_known_addr { host: "127.6.218.188" port: 42947 } }
22:11:28.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.626765 7300 sys_catalog.cc:455] T 00000000000000000000000000000000 P c4ceebdfcbbb41e3976917429d9ba525 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "44965596305b478e93a1a3c450737f25" member_type: VOTER last_known_addr { host: "127.6.218.190" port: 38037 } } peers { permanent_uuid: "c4ceebdfcbbb41e3976917429d9ba525" member_type: VOTER last_known_addr { host: "127.6.218.189" port: 41999 } } peers { permanent_uuid: "642848d3f5d14c9fa14e917104636c26" member_type: VOTER last_known_addr { host: "127.6.218.188" port: 42947 } } }
22:11:28.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.627538 7300 sys_catalog.cc:458] T 00000000000000000000000000000000 P c4ceebdfcbbb41e3976917429d9ba525 [sys.catalog]: This master's current role is: FOLLOWER
22:11:28.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.635444 7147 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "642848d3f5d14c9fa14e917104636c26" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c4ceebdfcbbb41e3976917429d9ba525"
22:11:28.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.636122 7147 raft_consensus.cc:3037] T 00000000000000000000000000000000 P c4ceebdfcbbb41e3976917429d9ba525 [term 0 FOLLOWER]: Advancing to term 1
22:11:28.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.642179 7148 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "642848d3f5d14c9fa14e917104636c26" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c4ceebdfcbbb41e3976917429d9ba525" is_pre_election: true
22:11:28.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.645725 7147 raft_consensus.cc:2454] T 00000000000000000000000000000000 P c4ceebdfcbbb41e3976917429d9ba525 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 642848d3f5d14c9fa14e917104636c26 in term 1.
22:11:28.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.646956 7146 raft_consensus.cc:1261] T 00000000000000000000000000000000 P c4ceebdfcbbb41e3976917429d9ba525 [term 1 FOLLOWER]: Refusing update from remote peer 642848d3f5d14c9fa14e917104636c26: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
22:11:28.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.653195 7282 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 642848d3f5d14c9fa14e917104636c26 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c4ceebdfcbbb41e3976917429d9ba525" member_type: VOTER last_known_addr { host: "127.6.218.189" port: 41999 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
22:11:28.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.654452 7173 sys_catalog.cc:564] T 00000000000000000000000000000000 P c4ceebdfcbbb41e3976917429d9ba525 [sys.catalog]: configured and running, proceeding with master startup.
22:11:28.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.706099 7300 sys_catalog.cc:455] T 00000000000000000000000000000000 P c4ceebdfcbbb41e3976917429d9ba525 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 642848d3f5d14c9fa14e917104636c26. Latest consensus state: current_term: 1 leader_uuid: "642848d3f5d14c9fa14e917104636c26" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "44965596305b478e93a1a3c450737f25" member_type: VOTER last_known_addr { host: "127.6.218.190" port: 38037 } } peers { permanent_uuid: "c4ceebdfcbbb41e3976917429d9ba525" member_type: VOTER last_known_addr { host: "127.6.218.189" port: 41999 } } peers { permanent_uuid: "642848d3f5d14c9fa14e917104636c26" member_type: VOTER last_known_addr { host: "127.6.218.188" port: 42947 } } }
22:11:28.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.706802 7300 sys_catalog.cc:458] T 00000000000000000000000000000000 P c4ceebdfcbbb41e3976917429d9ba525 [sys.catalog]: This master's current role is: FOLLOWER
22:11:28.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.707376 7306 sys_catalog.cc:455] T 00000000000000000000000000000000 P c4ceebdfcbbb41e3976917429d9ba525 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "642848d3f5d14c9fa14e917104636c26" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "44965596305b478e93a1a3c450737f25" member_type: VOTER last_known_addr { host: "127.6.218.190" port: 38037 } } peers { permanent_uuid: "c4ceebdfcbbb41e3976917429d9ba525" member_type: VOTER last_known_addr { host: "127.6.218.189" port: 41999 } } peers { permanent_uuid: "642848d3f5d14c9fa14e917104636c26" member_type: VOTER last_known_addr { host: "127.6.218.188" port: 42947 } } }
22:11:28.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.708272 7306 sys_catalog.cc:458] T 00000000000000000000000000000000 P c4ceebdfcbbb41e3976917429d9ba525 [sys.catalog]: This master's current role is: FOLLOWER
22:11:28.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.756000 7320 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
22:11:28.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:28.782640 7297 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
22:11:28.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:28.783171 7297 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
22:11:28.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:28.784025 7297 flags.cc:425] Enabled unsafe flag: --never_fsync=true
22:11:28.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:28.831982 7297 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
22:11:28.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:28.833627 7297 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.6.218.129
22:11:28.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.896046 7297 tablet_server_runner.cc:78] Tablet server non-default flags:
22:11:28.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:11:28.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.218.148:45245
22:11:28.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:11:28.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:11:28.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/ts-0/data
22:11:28.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/ts-0/wal
22:11:28.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:11:28.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:11:28.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.218.129:0
22:11:28.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
22:11:28.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:11:28.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:11:28.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/ts-0/data/info.pb
22:11:28.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.218.129
22:11:28.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:11:28.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.218.190:38037,127.6.218.189:41999,127.6.218.188:42947
22:11:28.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
22:11:28.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:11:28.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
22:11:28.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
22:11:28.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:11:28.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.218.129
22:11:28.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/ts-0/logs
22:11:28.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:11:28.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
22:11:28.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:11:28.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
22:11:28.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
22:11:28.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
22:11:28.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
22:11:28.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 21:22:24 UTC on ccycloud.quasar-wysbne.root.comops.site
22:11:28.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676318
22:11:28.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
22:11:28.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.898700 7297 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
22:11:28.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:28.900740 7297 file_cache.cc:493] Constructed file cache file cache with capacity 419430
22:11:28.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:28.925035 7322 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:11:29.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:29.130944 7282 sys_catalog.cc:455] T 00000000000000000000000000000000 P 642848d3f5d14c9fa14e917104636c26 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "642848d3f5d14c9fa14e917104636c26" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "44965596305b478e93a1a3c450737f25" member_type: VOTER last_known_addr { host: "127.6.218.190" port: 38037 } } peers { permanent_uuid: "c4ceebdfcbbb41e3976917429d9ba525" member_type: VOTER last_known_addr { host: "127.6.218.189" port: 41999 } } peers { permanent_uuid: "642848d3f5d14c9fa14e917104636c26" member_type: VOTER last_known_addr { host: "127.6.218.188" port: 42947 } } }
22:11:29.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:29.132035 7282 sys_catalog.cc:458] T 00000000000000000000000000000000 P 642848d3f5d14c9fa14e917104636c26 [sys.catalog]: This master's current role is: LEADER
22:11:29.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:29.225106 7290 catalog_manager.cc:1259] Loaded cluster ID: c2acd939602248d3834e64481eb92d3b
22:11:29.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:29.226205 7290 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 44965596305b478e93a1a3c450737f25: loading cluster ID for follower catalog manager: success
22:11:29.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:29.236433 7290 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 44965596305b478e93a1a3c450737f25: acquiring CA information for follower catalog manager: success
22:11:29.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:29.285982 7290 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 44965596305b478e93a1a3c450737f25: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
22:11:29.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:29.766955 7320 catalog_manager.cc:1259] Loaded cluster ID: c2acd939602248d3834e64481eb92d3b
22:11:29.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:29.767221 7320 catalog_manager.cc:1552] T 00000000000000000000000000000000 P c4ceebdfcbbb41e3976917429d9ba525: loading cluster ID for follower catalog manager: success
22:11:29.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:29.785079 7320 catalog_manager.cc:1574] T 00000000000000000000000000000000 P c4ceebdfcbbb41e3976917429d9ba525: acquiring CA information for follower catalog manager: success
22:11:29.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:29.801339 7320 catalog_manager.cc:1602] T 00000000000000000000000000000000 P c4ceebdfcbbb41e3976917429d9ba525: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
22:11:30.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:30.318964 7321 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 7297
22:11:30.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:30.438131 7297 thread.cc:636] GCE (cloud detector) Time spent creating pthread: real 1.515s user 0.372s sys 0.790s
22:11:30.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:30.441583 7297 thread.cc:602] GCE (cloud detector) Time spent starting thread: real 1.519s user 0.372s sys 0.790s
22:11:30.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:28.926646 7323 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:11:30.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:30.460817 7297 server_base.cc:1034] running on GCE node
22:11:30.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:30.462786 7331 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:11:30.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:30.464465 7297 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
22:11:30.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:30.475921 7297 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
22:11:30.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:30.477393 7297 hybrid_clock.cc:648] HybridClock initialized: now 1790547090477346 us; error 45 us; skew 500 ppm
22:11:30.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:30.478556 7297 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
22:11:30.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:30.507010 7297 webserver.cc:460] Webserver started at http://127.6.218.129:42757/ using document root <none> and password file <none>
22:11:30.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:30.508586 7297 fs_manager.cc:359] Metadata directory not provided
22:11:30.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:30.508833 7297 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
22:11:30.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:30.509338 7297 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
22:11:30.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:30.516131 7297 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/ts-0/data/instance:
22:11:30.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bccd6fff7a5c4ba7af4e5668f85af24b"
22:11:30.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 22:11:30 on dist-test-slave-qzth"
22:11:30.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:30.517660 7297 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/ts-0/wal/instance:
22:11:30.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bccd6fff7a5c4ba7af4e5668f85af24b"
22:11:30.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 22:11:30 on dist-test-slave-qzth"
22:11:30.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:30.532703 7297 fs_manager.cc:691] Time spent creating directory manager: real 0.014s user 0.002s sys 0.011s
22:11:30.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:30.545717 7336 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
22:11:30.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:30.546912 7297 fs_manager.cc:725] Time spent opening block manager: real 0.010s user 0.002s sys 0.008s
22:11:30.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:30.547276 7297 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/ts-0/data,/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/ts-0/wal
22:11:30.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bccd6fff7a5c4ba7af4e5668f85af24b"
22:11:30.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 22:11:30 on dist-test-slave-qzth"
22:11:30.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:30.547693 7297 fs_report.cc:383] FS layout report
22:11:30.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
22:11:30.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/ts-0/wal
22:11:30.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/ts-0/wal
22:11:30.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/ts-0/data/data
22:11:30.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
22:11:30.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
22:11:30.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
22:11:30.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
22:11:30.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
22:11:30.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
22:11:30.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
22:11:30.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
22:11:30.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
22:11:30.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
22:11:30.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
22:11:30.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:30.625056 7297 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
22:11:30.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:30.626991 7297 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
22:11:30.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:30.627548 7297 kserver.cc:163] Server-wide thread pool size limit: 3276
22:11:30.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:30.637867 7297 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
22:11:30.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:30.649806 7297 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
22:11:30.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:30.650094 7297 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.001s
22:11:30.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:30.650403 7297 ts_tablet_manager.cc:610] Registered 0 tablets
22:11:30.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:30.650604 7297 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
22:11:31.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:31.245743 7297 rpc_server.cc:307] RPC server started. Bound to: 127.6.218.129:40791
22:11:31.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:31.245903 7448 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.218.129:40791 every 8 connection(s)
22:11:31.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:31.249182 7297 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/ts-0/data/info.pb
22:11:31.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:31.253260 7018 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu as pid 7297
22:11:31.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:31.253721 7018 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/ts-0/wal/instance
22:11:31.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:31.267715 7018 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu
22:11:31.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu
22:11:31.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/ts-1/wal
22:11:31.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/ts-1/data
22:11:31.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
22:11:31.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
22:11:31.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
22:11:31.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:11:31.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:11:31.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:11:31.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/ts-1/logs
22:11:31.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/ts-1/data/info.pb
22:11:31.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:11:31.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
22:11:31.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
22:11:31.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
22:11:31.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
22:11:31.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:11:31.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:11:31.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:11:31.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
22:11:31.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
22:11:31.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.218.130:0
22:11:31.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.218.130
22:11:31.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.218.130
22:11:31.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:11:31.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.218.190:38037,127.6.218.189:41999,127.6.218.188:42947
22:11:31.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.218.148:45245
22:11:31.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:11:31.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:11:31.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
22:11:31.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:31.409936 7451 heartbeater.cc:347] Connected to a master server at 127.6.218.189:41999
22:11:31.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:31.410418 7451 heartbeater.cc:464] Registering TS with master...
22:11:31.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:31.411442 7451 heartbeater.cc:511] Master 127.6.218.189:41999 requested a full tablet report, sending...
22:11:31.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:31.415321 7138 ts_manager.cc:194] Registered new tserver with Master: bccd6fff7a5c4ba7af4e5668f85af24b (127.6.218.129:40791)
22:11:31.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:31.424440 7449 heartbeater.cc:347] Connected to a master server at 127.6.218.188:42947
22:11:31.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:31.424702 7449 heartbeater.cc:464] Registering TS with master...
22:11:31.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:31.425419 7449 heartbeater.cc:511] Master 127.6.218.188:42947 requested a full tablet report, sending...
22:11:31.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:31.431553 7217 ts_manager.cc:194] Registered new tserver with Master: bccd6fff7a5c4ba7af4e5668f85af24b (127.6.218.129:40791)
22:11:31.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:31.434574 7217 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.6.218.129:48137
22:11:31.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:31.457810 7454 heartbeater.cc:347] Connected to a master server at 127.6.218.190:38037
22:11:31.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:31.458170 7454 heartbeater.cc:464] Registering TS with master...
22:11:31.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:31.458794 7454 heartbeater.cc:511] Master 127.6.218.190:38037 requested a full tablet report, sending...
22:11:31.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:31.461932 7062 ts_manager.cc:194] Registered new tserver with Master: bccd6fff7a5c4ba7af4e5668f85af24b (127.6.218.129:40791)
22:11:31.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:31.807062 7450 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
22:11:31.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:31.807590 7450 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
22:11:31.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:31.808208 7450 flags.cc:425] Enabled unsafe flag: --never_fsync=true
22:11:31.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:31.840360 7450 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
22:11:31.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:31.841271 7450 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.6.218.130
22:11:31.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:31.876145 7450 tablet_server_runner.cc:78] Tablet server non-default flags:
22:11:31.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:11:31.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.218.148:45245
22:11:31.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:11:31.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:11:31.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/ts-1/data
22:11:31.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/ts-1/wal
22:11:31.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:11:31.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:11:31.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.218.130:0
22:11:31.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
22:11:31.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:11:31.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:11:31.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/ts-1/data/info.pb
22:11:31.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.218.130
22:11:31.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:11:31.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.218.190:38037,127.6.218.189:41999,127.6.218.188:42947
22:11:31.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
22:11:31.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:11:31.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
22:11:31.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
22:11:31.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:11:31.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.218.130
22:11:31.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/ts-1/logs
22:11:31.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:11:31.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
22:11:31.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:11:31.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
22:11:31.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
22:11:31.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
22:11:31.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
22:11:31.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 21:22:24 UTC on ccycloud.quasar-wysbne.root.comops.site
22:11:31.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676318
22:11:31.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
22:11:31.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:31.877573 7450 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
22:11:31.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:31.879570 7450 file_cache.cc:493] Constructed file cache file cache with capacity 419430
22:11:31.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:31.904301 7466 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:11:32.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:32.439005 7449 heartbeater.cc:503] Master 127.6.218.188:42947 was elected leader, sending a full tablet report...
22:11:33.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:31.904783 7468 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:11:33.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:31.907352 7450 server_base.cc:1034] running on GCE node
22:11:33.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:31.909894 7465 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:11:33.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:33.097533 7450 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
22:11:33.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:33.099881 7450 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
22:11:33.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:33.101258 7450 hybrid_clock.cc:648] HybridClock initialized: now 1790547093101223 us; error 38 us; skew 500 ppm
22:11:33.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:33.102123 7450 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
22:11:33.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:33.108283 7450 webserver.cc:460] Webserver started at http://127.6.218.130:35999/ using document root <none> and password file <none>
22:11:33.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:33.109236 7450 fs_manager.cc:359] Metadata directory not provided
22:11:33.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:33.109483 7450 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
22:11:33.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:33.110034 7450 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
22:11:33.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:33.115075 7450 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/ts-1/data/instance:
22:11:33.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "159b8297a0c74e8fb9f288998edd085e"
22:11:33.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 22:11:33 on dist-test-slave-qzth"
22:11:33.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:33.116196 7450 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/ts-1/wal/instance:
22:11:33.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "159b8297a0c74e8fb9f288998edd085e"
22:11:33.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 22:11:33 on dist-test-slave-qzth"
22:11:33.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:33.123486 7450 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.001s sys 0.008s
22:11:33.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:33.128789 7477 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
22:11:33.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:33.129786 7450 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.001s sys 0.001s
22:11:33.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:33.130198 7450 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/ts-1/data,/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/ts-1/wal
22:11:33.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "159b8297a0c74e8fb9f288998edd085e"
22:11:33.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 22:11:33 on dist-test-slave-qzth"
22:11:33.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:33.130605 7450 fs_report.cc:383] FS layout report
22:11:33.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
22:11:33.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/ts-1/wal
22:11:33.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/ts-1/wal
22:11:33.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/ts-1/data/data
22:11:33.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
22:11:33.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
22:11:33.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
22:11:33.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
22:11:33.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
22:11:33.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
22:11:33.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
22:11:33.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
22:11:33.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
22:11:33.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
22:11:33.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
22:11:33.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:33.163591 7450 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
22:11:33.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:33.165074 7450 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
22:11:33.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:33.165570 7450 kserver.cc:163] Server-wide thread pool size limit: 3276
22:11:33.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:33.168043 7450 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
22:11:33.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:33.172055 7450 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
22:11:33.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:33.172284 7450 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
22:11:33.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:33.172636 7450 ts_tablet_manager.cc:610] Registered 0 tablets
22:11:33.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:33.172866 7450 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
22:11:33.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:33.344357 7450 rpc_server.cc:307] RPC server started. Bound to: 127.6.218.130:44667
22:11:33.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:33.344411 7589 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.218.130:44667 every 8 connection(s)
22:11:33.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:33.347812 7450 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/ts-1/data/info.pb
22:11:33.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:33.357275 7018 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu as pid 7450
22:11:33.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:33.358008 7018 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/ts-1/wal/instance
22:11:33.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:33.374394 7018 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu
22:11:33.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu
22:11:33.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/ts-2/wal
22:11:33.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/ts-2/data
22:11:33.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
22:11:33.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
22:11:33.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
22:11:33.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:11:33.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:11:33.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:11:33.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/ts-2/logs
22:11:33.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/ts-2/data/info.pb
22:11:33.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:11:33.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
22:11:33.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
22:11:33.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
22:11:33.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
22:11:33.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:11:33.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:11:33.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:11:33.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
22:11:33.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
22:11:33.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.218.131:0
22:11:33.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.218.131
22:11:33.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.218.131
22:11:33.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:11:33.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.218.190:38037,127.6.218.189:41999,127.6.218.188:42947
22:11:33.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.218.148:45245
22:11:33.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:11:33.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:11:33.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
22:11:33.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:33.393962 7590 heartbeater.cc:347] Connected to a master server at 127.6.218.188:42947
22:11:33.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:33.394374 7590 heartbeater.cc:464] Registering TS with master...
22:11:33.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:33.395421 7590 heartbeater.cc:511] Master 127.6.218.188:42947 requested a full tablet report, sending...
22:11:33.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:33.397783 7217 ts_manager.cc:194] Registered new tserver with Master: 159b8297a0c74e8fb9f288998edd085e (127.6.218.130:44667)
22:11:33.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:33.401136 7217 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.6.218.130:60759
22:11:33.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:33.408233 7592 heartbeater.cc:347] Connected to a master server at 127.6.218.190:38037
22:11:33.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:33.408556 7592 heartbeater.cc:464] Registering TS with master...
22:11:33.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:33.409479 7592 heartbeater.cc:511] Master 127.6.218.190:38037 requested a full tablet report, sending...
22:11:33.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:33.412288 7062 ts_manager.cc:194] Registered new tserver with Master: 159b8297a0c74e8fb9f288998edd085e (127.6.218.130:44667)
22:11:33.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:33.413961 7591 heartbeater.cc:347] Connected to a master server at 127.6.218.189:41999
22:11:33.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:33.414304 7591 heartbeater.cc:464] Registering TS with master...
22:11:33.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:33.415378 7591 heartbeater.cc:511] Master 127.6.218.189:41999 requested a full tablet report, sending...
22:11:33.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:33.417693 7138 ts_manager.cc:194] Registered new tserver with Master: 159b8297a0c74e8fb9f288998edd085e (127.6.218.130:44667)
22:11:33.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:33.699512 7600 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
22:11:33.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:33.699932 7600 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
22:11:33.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:33.700322 7600 flags.cc:425] Enabled unsafe flag: --never_fsync=true
22:11:33.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:33.730512 7600 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
22:11:33.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:33.731387 7600 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.6.218.131
22:11:33.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:33.765489 7600 tablet_server_runner.cc:78] Tablet server non-default flags:
22:11:33.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:11:33.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.218.148:45245
22:11:33.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:11:33.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:11:33.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/ts-2/data
22:11:33.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/ts-2/wal
22:11:33.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:11:33.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:11:33.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.218.131:0
22:11:33.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
22:11:33.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:11:33.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:11:33.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/ts-2/data/info.pb
22:11:33.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.218.131
22:11:33.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:11:33.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.218.190:38037,127.6.218.189:41999,127.6.218.188:42947
22:11:33.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
22:11:33.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:11:33.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
22:11:33.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
22:11:33.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:11:33.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.218.131
22:11:33.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/ts-2/logs
22:11:33.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:11:33.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
22:11:33.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:11:33.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
22:11:33.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
22:11:33.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
22:11:33.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
22:11:33.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 21:22:24 UTC on ccycloud.quasar-wysbne.root.comops.site
22:11:33.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676318
22:11:33.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
22:11:33.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:33.766646 7600 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
22:11:33.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:33.768100 7600 file_cache.cc:493] Constructed file cache file cache with capacity 419430
22:11:33.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:33.788012 7608 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:11:34.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:34.410569 7590 heartbeater.cc:503] Master 127.6.218.188:42947 was elected leader, sending a full tablet report...
22:11:34.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:33.788054 7607 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:11:34.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:33.788832 7610 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:11:34.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:33.790542 7600 server_base.cc:1034] running on GCE node
22:11:34.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:34.957368 7600 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
22:11:34.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:34.959678 7600 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
22:11:34.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:34.961045 7600 hybrid_clock.cc:648] HybridClock initialized: now 1790547094961015 us; error 42 us; skew 500 ppm
22:11:34.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:34.961946 7600 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
22:11:34.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:34.975889 7600 webserver.cc:460] Webserver started at http://127.6.218.131:34301/ using document root <none> and password file <none>
22:11:34.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:34.976709 7600 fs_manager.cc:359] Metadata directory not provided
22:11:34.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:34.976897 7600 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
22:11:34.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:34.977339 7600 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
22:11:34.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:34.982481 7600 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/ts-2/data/instance:
22:11:34.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "82062b22579a449f8df03a499c686f51"
22:11:34.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 22:11:34 on dist-test-slave-qzth"
22:11:34.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:34.983505 7600 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/ts-2/wal/instance:
22:11:34.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "82062b22579a449f8df03a499c686f51"
22:11:34.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 22:11:34 on dist-test-slave-qzth"
22:11:34.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:34.990821 7600 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.003s sys 0.005s
22:11:34.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:34.995875 7619 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
22:11:34.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:34.996841 7600 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.005s sys 0.000s
22:11:34.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:34.997103 7600 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/ts-2/data,/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/ts-2/wal
22:11:34.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "82062b22579a449f8df03a499c686f51"
22:11:34.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 22:11:34 on dist-test-slave-qzth"
22:11:34.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:34.997368 7600 fs_report.cc:383] FS layout report
22:11:34.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
22:11:34.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/ts-2/wal
22:11:34.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/ts-2/wal
22:11:34.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/ts-2/data/data
22:11:34.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
22:11:34.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
22:11:34.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
22:11:34.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
22:11:34.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
22:11:34.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
22:11:34.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
22:11:34.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
22:11:34.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
22:11:34.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
22:11:34.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
22:11:35.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:35.048432 7600 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
22:11:35.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:35.050302 7600 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
22:11:35.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:35.050832 7600 kserver.cc:163] Server-wide thread pool size limit: 3276
22:11:35.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:35.054152 7600 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
22:11:35.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:35.059525 7600 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
22:11:35.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:35.060638 7600 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.001s user 0.001s sys 0.000s
22:11:35.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:35.060936 7600 ts_tablet_manager.cc:610] Registered 0 tablets
22:11:35.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:35.061131 7600 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
22:11:35.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:35.210466 7600 rpc_server.cc:307] RPC server started. Bound to: 127.6.218.131:34259
22:11:35.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:35.210561 7731 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.218.131:34259 every 8 connection(s)
22:11:35.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:35.213168 7600 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/ts-2/data/info.pb
22:11:35.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:35.223752 7018 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu as pid 7600
22:11:35.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:35.224280 7018 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster5500451891160507557/ts-2/wal/instance
22:11:35.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:35.258805 7732 heartbeater.cc:347] Connected to a master server at 127.6.218.188:42947
22:11:35.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:35.259202 7732 heartbeater.cc:464] Registering TS with master...
22:11:35.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:35.260341 7732 heartbeater.cc:511] Master 127.6.218.188:42947 requested a full tablet report, sending...
22:11:35.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:35.263466 7217 ts_manager.cc:194] Registered new tserver with Master: 82062b22579a449f8df03a499c686f51 (127.6.218.131:34259)
22:11:35.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:35.265727 7217 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.6.218.131:50119
22:11:35.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:35.269356 7734 heartbeater.cc:347] Connected to a master server at 127.6.218.190:38037
22:11:35.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:35.269698 7734 heartbeater.cc:464] Registering TS with master...
22:11:35.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:35.270251 7733 heartbeater.cc:347] Connected to a master server at 127.6.218.189:41999
22:11:35.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:35.270551 7733 heartbeater.cc:464] Registering TS with master...
22:11:35.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:35.270613 7734 heartbeater.cc:511] Master 127.6.218.190:38037 requested a full tablet report, sending...
22:11:35.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:35.273231 7733 heartbeater.cc:511] Master 127.6.218.189:41999 requested a full tablet report, sending...
22:11:35.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:35.274389 7063 ts_manager.cc:194] Registered new tserver with Master: 82062b22579a449f8df03a499c686f51 (127.6.218.131:34259)
22:11:35.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:35.275825 7138 ts_manager.cc:194] Registered new tserver with Master: 82062b22579a449f8df03a499c686f51 (127.6.218.131:34259)
22:11:35.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:35.289741 7018 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
22:11:35.300 [DEBUG - main] (MiniKuduCluster.java:195) Response:
22:11:35.301 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
22:11:35.313 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.6.218.190"
port: 38037
}
bound_http_address {
host: "127.6.218.190"
port: 41985
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.6.218.189"
port: 41999
}
bound_http_address {
host: "127.6.218.189"
port: 34747
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.6.218.188"
port: 42947
}
bound_http_address {
host: "127.6.218.188"
port: 44119
}
}
}
22:11:35.318 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
22:11:35.321 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.6.218.129"
port: 40791
}
bound_http_address {
host: "127.6.218.129"
port: 42757
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.6.218.130"
port: 44667
}
bound_http_address {
host: "127.6.218.130"
port: 35999
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.6.218.131"
port: 34259
}
bound_http_address {
host: "127.6.218.131"
port: 34301
}
}
}
22:11:35.321 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
22:11:35.489 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
22:11:35.603 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskyO6VHJ/test-tmp
22:11:35.604 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
22:11:35.604 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskyO6VHJ/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
22:11:35.607 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
22:11:35.929 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
22:11:35.929 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
22:11:35.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:35.932219 7743 mini_oidc.cc:152] Starting JWKS server
22:11:35.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:35.935981 7743 webserver.cc:460] Webserver started at http://127.0.0.1:34207/ using document root <none> and password file <none>
22:11:35.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:35.936363 7743 mini_oidc.cc:189] Starting OIDC Discovery server
22:11:35.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:35.938028 7743 webserver.cc:460] Webserver started at http://127.0.0.1:41827/ using document root <none> and password file <none>
22:11:36.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
22:11:36.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/krb5kdc/principal' for realm 'KRBTEST.COM',
22:11:36.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
22:11:36.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 22:11:36 dist-test-slave-qzth krb5kdc[7755](info): setting up network...
22:11:36.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
22:11:36.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 22:11:36 dist-test-slave-qzth krb5kdc[7755](info): set up 2 sockets
22:11:36.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 22:11:36 dist-test-slave-qzth krb5kdc[7755](info): commencing operation
22:11:36.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
22:11:36.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:36.272949 7732 heartbeater.cc:503] Master 127.6.218.188:42947 was elected leader, sending a full tablet report...
22:11:37.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:37.024581 7743 mini_kdc.cc:121] Time spent starting KDC: real 1.061s user 0.001s sys 0.017s
22:11:37.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
22:11:37.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
22:11:37.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
22:11:37.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
22:11:37.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
22:11:37.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
22:11:37.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
22:11:37.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
22:11:37.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
22:11:37.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
22:11:37.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/krb5kdc/test-user.keytab.
22:11:37.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/krb5kdc/test-user.keytab.
22:11:37.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 22:11:37 dist-test-slave-qzth krb5kdc[7755](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790547097, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
22:11:37.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
22:11:37.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T22:11:37Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
22:11:37.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T22:11:37Z Disabled control of system clock
22:11:37.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.7.143.254@KRBTEST.COM; defaulting to no policy
22:11:37.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
22:11:37.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.7.143.254@KRBTEST.COM" created.
22:11:37.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
22:11:37.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.7.143.254 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/krb5kdc/kudu_127.7.143.254.keytab.
22:11:37.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.7.143.254 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/krb5kdc/kudu_127.7.143.254.keytab.
22:11:37.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:37.210572 7743 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu
22:11:37.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu
22:11:37.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-0/wal
22:11:37.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-0/data
22:11:37.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
22:11:37.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
22:11:37.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
22:11:37.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:11:37.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:11:37.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:11:37.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-0/logs
22:11:37.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-0/data/info.pb
22:11:37.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:11:37.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
22:11:37.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
22:11:37.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
22:11:37.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
22:11:37.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:11:37.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:11:37.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:11:37.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
22:11:37.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
22:11:37.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
22:11:37.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
22:11:37.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.143.254:40351
22:11:37.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.143.254
22:11:37.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:11:37.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/krb5kdc/kudu_127.7.143.254.keytab
22:11:37.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.143.254
22:11:37.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
22:11:37.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
22:11:37.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
22:11:37.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.143.212:44837
22:11:37.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:11:37.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:11:37.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:11:37.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
22:11:37.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.143.254:40351,127.7.143.253:45437,127.7.143.252:41589
22:11:37.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
22:11:37.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34207/jwks.json
22:11:37.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
22:11:37.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:37.517856 7769 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
22:11:37.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:37.518563 7769 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
22:11:37.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:37.519403 7769 flags.cc:425] Enabled unsafe flag: --never_fsync=true
22:11:37.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:37.550144 7769 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
22:11:37.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:37.550340 7769 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
22:11:37.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:37.550674 7769 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
22:11:37.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:37.550874 7769 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
22:11:37.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:37.551054 7769 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
22:11:37.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:37.551240 7769 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
22:11:37.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:37.551363 7769 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34207/jwks.json
22:11:37.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:37.585930 7769 master_runner.cc:386] Master server non-default flags:
22:11:37.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:11:37.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.143.212:44837
22:11:37.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:11:37.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:11:37.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-0/data
22:11:37.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-0/wal
22:11:37.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
22:11:37.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
22:11:37.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.143.254:40351,127.7.143.253:45437,127.7.143.252:41589
22:11:37.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/krb5kdc/kudu_127.7.143.254.keytab
22:11:37.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.143.254
22:11:37.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:11:37.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:11:37.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
22:11:37.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.143.254:40351
22:11:37.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
22:11:37.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
22:11:37.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
22:11:37.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34207/jwks.json
22:11:37.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
22:11:37.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
22:11:37.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
22:11:37.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:11:37.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:11:37.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-0/data/info.pb
22:11:37.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.143.254
22:11:37.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:11:37.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
22:11:37.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:11:37.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
22:11:37.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
22:11:37.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:11:37.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-0/logs
22:11:37.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:11:37.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
22:11:37.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:11:37.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
22:11:37.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
22:11:37.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
22:11:37.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
22:11:37.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 21:22:24 UTC on ccycloud.quasar-wysbne.root.comops.site
22:11:37.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676318
22:11:37.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
22:11:37.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:37.587176 7769 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
22:11:37.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:37.588477 7769 file_cache.cc:493] Constructed file cache file cache with capacity 419430
22:11:37.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:37.607558 7778 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:11:38.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:37.607565 7775 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:11:38.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:37.607687 7776 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:11:38.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:38.803761 7777 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
22:11:38.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:38.803865 7769 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
22:11:38.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:38.808079 7769 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
22:11:38.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:38.811148 7769 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
22:11:38.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:38.812583 7769 hybrid_clock.cc:648] HybridClock initialized: now 1790547098812518 us; error 80 us; skew 500 ppm
22:11:38.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:38.813609 7769 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
22:11:38.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 22:11:38 dist-test-slave-qzth krb5kdc[7755](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790547098, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.143.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
22:11:38.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:38.820147 7769 init.cc:377] Logged in from keytab as kudu/127.7.143.254@KRBTEST.COM (short username kudu)
22:11:38.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:38.824499 7769 webserver.cc:460] Webserver started at http://127.7.143.254:45673/ using document root <none> and password file <none>
22:11:38.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:38.825387 7769 fs_manager.cc:359] Metadata directory not provided
22:11:38.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:38.825608 7769 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
22:11:38.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:38.826110 7769 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
22:11:38.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:38.830572 7769 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-0/data/instance:
22:11:38.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "929b15eec5ef4bd9adb23cb19aea80f2"
22:11:38.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 22:11:38 on dist-test-slave-qzth"
22:11:38.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:38.831648 7769 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-0/wal/instance:
22:11:38.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "929b15eec5ef4bd9adb23cb19aea80f2"
22:11:38.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 22:11:38 on dist-test-slave-qzth"
22:11:38.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:38.838537 7769 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.006s sys 0.001s
22:11:38.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:38.843917 7787 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
22:11:38.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:38.844877 7769 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.002s
22:11:38.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:38.845232 7769 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-0/data,/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-0/wal
22:11:38.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "929b15eec5ef4bd9adb23cb19aea80f2"
22:11:38.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 22:11:38 on dist-test-slave-qzth"
22:11:38.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:38.845546 7769 fs_report.cc:383] FS layout report
22:11:38.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
22:11:38.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-0/wal
22:11:38.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-0/wal
22:11:38.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-0/data/data
22:11:38.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
22:11:38.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
22:11:38.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
22:11:38.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
22:11:38.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
22:11:38.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
22:11:38.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
22:11:38.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
22:11:38.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
22:11:38.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
22:11:38.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
22:11:38.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:38.911368 7769 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
22:11:38.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:38.917369 7769 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
22:11:38.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:38.917881 7769 kserver.cc:163] Server-wide thread pool size limit: 3276
22:11:38.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:38.992053 7769 rpc_server.cc:307] RPC server started. Bound to: 127.7.143.254:40351
22:11:38.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:38.992153 7838 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.143.254:40351 every 8 connection(s)
22:11:38.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:38.994699 7769 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-0/data/info.pb
22:11:39.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:39.000092 7743 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu as pid 7769
22:11:39.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:39.000552 7743 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-0/wal/instance
22:11:39.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:39.000988 7839 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
22:11:39.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:39.013301 7839 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.143.254" port: 40351 } has no permanent_uuid. Determining permanent_uuid...
22:11:39.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.7.143.253@KRBTEST.COM; defaulting to no policy
22:11:39.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
22:11:39.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.7.143.253@KRBTEST.COM" created.
22:11:39.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 22:11:39 dist-test-slave-qzth krb5kdc[7755](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790547098, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.143.254@KRBTEST.COM for kudu/127.7.143.254@KRBTEST.COM
22:11:39.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
22:11:39.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.7.143.253 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/krb5kdc/kudu_127.7.143.253.keytab.
22:11:39.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.7.143.253 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/krb5kdc/kudu_127.7.143.253.keytab.
22:11:39.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:39.044313 7743 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu
22:11:39.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu
22:11:39.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-1/wal
22:11:39.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-1/data
22:11:39.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
22:11:39.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
22:11:39.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
22:11:39.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:11:39.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:11:39.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:11:39.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-1/logs
22:11:39.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-1/data/info.pb
22:11:39.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:11:39.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
22:11:39.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
22:11:39.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
22:11:39.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
22:11:39.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:11:39.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:11:39.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:11:39.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
22:11:39.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
22:11:39.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
22:11:39.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
22:11:39.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.143.253:45437
22:11:39.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.143.253
22:11:39.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:11:39.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/krb5kdc/kudu_127.7.143.253.keytab
22:11:39.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.143.253
22:11:39.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
22:11:39.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
22:11:39.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
22:11:39.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.143.212:44837
22:11:39.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:11:39.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:11:39.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:11:39.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
22:11:39.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.143.254:40351,127.7.143.253:45437,127.7.143.252:41589
22:11:39.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
22:11:39.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34207/jwks.json
22:11:39.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
22:11:39.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:39.053694 7839 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.143.253" port: 45437 } has no permanent_uuid. Determining permanent_uuid...
22:11:39.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:39.056298 7788 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.143.253:45437: connect: Connection refused (error 111)
22:11:39.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:39.060901 7839 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.143.253:45437: Network error: Client connection negotiation failed: client connection to 127.7.143.253:45437: connect: Connection refused (error 111)
22:11:39.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:39.110428 7839 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.143.253" port: 45437 } attempt: 1
22:11:39.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:39.114311 7839 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.143.253:45437: Network error: Client connection negotiation failed: client connection to 127.7.143.253:45437: connect: Connection refused (error 111)
22:11:39.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:39.182696 7839 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.143.253" port: 45437 } attempt: 2
22:11:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:39.186553 7839 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.143.253:45437: Network error: Client connection negotiation failed: client connection to 127.7.143.253:45437: connect: Connection refused (error 111)
22:11:39.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:39.277902 7839 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.143.253" port: 45437 } attempt: 3
22:11:39.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:39.281575 7839 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.143.253:45437: Network error: Client connection negotiation failed: client connection to 127.7.143.253:45437: connect: Connection refused (error 111)
22:11:39.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:39.370916 7845 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
22:11:39.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:39.371585 7845 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
22:11:39.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:39.372334 7845 flags.cc:425] Enabled unsafe flag: --never_fsync=true
22:11:39.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:39.406893 7845 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
22:11:39.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:39.407231 7845 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
22:11:39.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:39.407671 7845 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
22:11:39.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:39.407999 7845 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
22:11:39.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:39.408325 7845 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
22:11:39.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:39.408638 7845 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
22:11:39.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:39.408879 7845 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34207/jwks.json
22:11:39.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:39.424921 7839 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.143.253" port: 45437 } attempt: 4
22:11:39.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:39.429121 7839 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.143.253:45437: Network error: Client connection negotiation failed: client connection to 127.7.143.253:45437: connect: Connection refused (error 111)
22:11:39.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:39.452581 7845 master_runner.cc:386] Master server non-default flags:
22:11:39.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:11:39.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.143.212:44837
22:11:39.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:11:39.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:11:39.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-1/data
22:11:39.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-1/wal
22:11:39.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
22:11:39.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
22:11:39.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.143.254:40351,127.7.143.253:45437,127.7.143.252:41589
22:11:39.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/krb5kdc/kudu_127.7.143.253.keytab
22:11:39.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.143.253
22:11:39.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:11:39.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:11:39.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
22:11:39.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.143.253:45437
22:11:39.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
22:11:39.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
22:11:39.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
22:11:39.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34207/jwks.json
22:11:39.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
22:11:39.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
22:11:39.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
22:11:39.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:11:39.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:11:39.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-1/data/info.pb
22:11:39.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.143.253
22:11:39.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:11:39.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
22:11:39.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:11:39.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
22:11:39.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
22:11:39.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:11:39.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-1/logs
22:11:39.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:11:39.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
22:11:39.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:11:39.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
22:11:39.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
22:11:39.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
22:11:39.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
22:11:39.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 21:22:24 UTC on ccycloud.quasar-wysbne.root.comops.site
22:11:39.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676318
22:11:39.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
22:11:39.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:39.454154 7845 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
22:11:39.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:39.455540 7845 file_cache.cc:493] Constructed file cache file cache with capacity 419430
22:11:39.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:39.474620 7852 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:11:39.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:39.728523 7839 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.143.253" port: 45437 } attempt: 5
22:11:39.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:39.732642 7839 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.143.253:45437: Network error: Client connection negotiation failed: client connection to 127.7.143.253:45437: connect: Connection refused (error 111)
22:11:40.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:40.280272 7839 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.143.253" port: 45437 } attempt: 6
22:11:40.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:40.287690 7839 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.143.253:45437: Network error: Client connection negotiation failed: client connection to 127.7.143.253:45437: connect: Connection refused (error 111)
22:11:40.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:39.474627 7855 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:11:40.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:39.474932 7853 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:11:40.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:39.477252 7845 server_base.cc:1034] running on GCE node
22:11:40.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:40.700325 7845 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
22:11:40.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:40.702816 7845 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
22:11:40.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:40.704197 7845 hybrid_clock.cc:648] HybridClock initialized: now 1790547100704143 us; error 57 us; skew 500 ppm
22:11:40.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:40.705112 7845 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
22:11:40.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 22:11:40 dist-test-slave-qzth krb5kdc[7755](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790547100, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.143.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
22:11:40.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:40.716331 7845 init.cc:377] Logged in from keytab as kudu/127.7.143.253@KRBTEST.COM (short username kudu)
22:11:40.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:40.725109 7845 webserver.cc:460] Webserver started at http://127.7.143.253:35067/ using document root <none> and password file <none>
22:11:40.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:40.725996 7845 fs_manager.cc:359] Metadata directory not provided
22:11:40.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:40.726167 7845 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
22:11:40.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:40.726529 7845 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
22:11:40.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:40.731237 7845 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-1/data/instance:
22:11:40.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "65fbe4442b8b4d21af2fa7d6b162abb7"
22:11:40.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 22:11:40 on dist-test-slave-qzth"
22:11:40.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:40.732383 7845 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-1/wal/instance:
22:11:40.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "65fbe4442b8b4d21af2fa7d6b162abb7"
22:11:40.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 22:11:40 on dist-test-slave-qzth"
22:11:40.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:40.739415 7845 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.001s sys 0.007s
22:11:40.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:40.744529 7865 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
22:11:40.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:40.745527 7845 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
22:11:40.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:40.745796 7845 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-1/data,/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-1/wal
22:11:40.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "65fbe4442b8b4d21af2fa7d6b162abb7"
22:11:40.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 22:11:40 on dist-test-slave-qzth"
22:11:40.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:40.746138 7845 fs_report.cc:383] FS layout report
22:11:40.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
22:11:40.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-1/wal
22:11:40.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-1/wal
22:11:40.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-1/data/data
22:11:40.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
22:11:40.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
22:11:40.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
22:11:40.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
22:11:40.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
22:11:40.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
22:11:40.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
22:11:40.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
22:11:40.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
22:11:40.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
22:11:40.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
22:11:40.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:40.788033 7845 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
22:11:40.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:40.793592 7845 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
22:11:40.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:40.794138 7845 kserver.cc:163] Server-wide thread pool size limit: 3276
22:11:40.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:40.871096 7845 rpc_server.cc:307] RPC server started. Bound to: 127.7.143.253:45437
22:11:40.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:40.871145 7916 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.143.253:45437 every 8 connection(s)
22:11:40.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:40.873653 7845 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-1/data/info.pb
22:11:40.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:40.879557 7743 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu as pid 7845
22:11:40.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:40.879887 7743 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-1/wal/instance
22:11:40.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:40.880352 7917 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
22:11:40.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:40.890961 7917 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.143.254" port: 40351 } has no permanent_uuid. Determining permanent_uuid...
22:11:40.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.7.143.252@KRBTEST.COM; defaulting to no policy
22:11:40.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
22:11:40.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.7.143.252@KRBTEST.COM" created.
22:11:40.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 22:11:40 dist-test-slave-qzth krb5kdc[7755](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790547100, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.143.253@KRBTEST.COM for kudu/127.7.143.254@KRBTEST.COM
22:11:40.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
22:11:40.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.7.143.252 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/krb5kdc/kudu_127.7.143.252.keytab.
22:11:40.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.7.143.252 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/krb5kdc/kudu_127.7.143.252.keytab.
22:11:40.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:40.918116 7743 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu
22:11:40.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu
22:11:40.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-2/wal
22:11:40.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-2/data
22:11:40.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
22:11:40.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
22:11:40.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
22:11:40.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:11:40.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:11:40.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:11:40.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-2/logs
22:11:40.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-2/data/info.pb
22:11:40.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:11:40.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
22:11:40.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
22:11:40.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
22:11:40.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
22:11:40.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:11:40.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:11:40.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:11:40.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
22:11:40.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
22:11:40.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
22:11:40.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
22:11:40.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.143.252:41589
22:11:40.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.143.252
22:11:40.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:11:40.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/krb5kdc/kudu_127.7.143.252.keytab
22:11:40.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.143.252
22:11:40.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
22:11:40.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
22:11:40.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
22:11:40.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.143.212:44837
22:11:40.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:11:40.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:11:40.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:11:40.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
22:11:40.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.143.254:40351,127.7.143.253:45437,127.7.143.252:41589
22:11:40.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
22:11:40.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34207/jwks.json
22:11:40.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
22:11:40.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:40.927634 7917 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.143.253" port: 45437 } has no permanent_uuid. Determining permanent_uuid...
22:11:40.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 22:11:40 dist-test-slave-qzth krb5kdc[7755](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790547100, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.143.253@KRBTEST.COM for kudu/127.7.143.253@KRBTEST.COM
22:11:40.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:40.963058 7917 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.143.252" port: 41589 } has no permanent_uuid. Determining permanent_uuid...
22:11:40.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:40.965739 7869 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.143.252:41589: connect: Connection refused (error 111)
22:11:40.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:40.980151 7917 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.143.252:41589: Network error: Client connection negotiation failed: client connection to 127.7.143.252:41589: connect: Connection refused (error 111)
22:11:41.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:41.029573 7917 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.143.252" port: 41589 } attempt: 1
22:11:41.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:41.032913 7917 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.143.252:41589: Network error: Client connection negotiation failed: client connection to 127.7.143.252:41589: connect: Connection refused (error 111)
22:11:41.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:41.101377 7917 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.143.252" port: 41589 } attempt: 2
22:11:41.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:41.105154 7917 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.143.252:41589: Network error: Client connection negotiation failed: client connection to 127.7.143.252:41589: connect: Connection refused (error 111)
22:11:41.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:41.196563 7917 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.143.252" port: 41589 } attempt: 3
22:11:41.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:41.200492 7917 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.143.252:41589: Network error: Client connection negotiation failed: client connection to 127.7.143.252:41589: connect: Connection refused (error 111)
22:11:41.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:41.260175 7923 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
22:11:41.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:41.260654 7923 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
22:11:41.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:41.261179 7923 flags.cc:425] Enabled unsafe flag: --never_fsync=true
22:11:41.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:41.291311 7923 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
22:11:41.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:41.291581 7923 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
22:11:41.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:41.292016 7923 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
22:11:41.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:41.292357 7923 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
22:11:41.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:41.292634 7923 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
22:11:41.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:41.292949 7923 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
22:11:41.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:41.293388 7923 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34207/jwks.json
22:11:41.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:41.331269 7923 master_runner.cc:386] Master server non-default flags:
22:11:41.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:11:41.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.143.212:44837
22:11:41.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:11:41.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:11:41.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-2/data
22:11:41.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-2/wal
22:11:41.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
22:11:41.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
22:11:41.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.143.254:40351,127.7.143.253:45437,127.7.143.252:41589
22:11:41.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/krb5kdc/kudu_127.7.143.252.keytab
22:11:41.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.143.252
22:11:41.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:11:41.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:11:41.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
22:11:41.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.143.252:41589
22:11:41.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
22:11:41.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
22:11:41.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
22:11:41.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34207/jwks.json
22:11:41.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
22:11:41.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
22:11:41.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
22:11:41.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:11:41.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:11:41.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-2/data/info.pb
22:11:41.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.143.252
22:11:41.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:11:41.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
22:11:41.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:11:41.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
22:11:41.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
22:11:41.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:11:41.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-2/logs
22:11:41.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:11:41.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
22:11:41.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:11:41.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
22:11:41.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
22:11:41.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
22:11:41.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
22:11:41.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 21:22:24 UTC on ccycloud.quasar-wysbne.root.comops.site
22:11:41.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676318
22:11:41.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
22:11:41.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:41.332319 7923 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
22:11:41.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:41.333431 7923 file_cache.cc:493] Constructed file cache file cache with capacity 419430
22:11:41.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:41.343863 7917 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.143.252" port: 41589 } attempt: 4
22:11:41.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:41.347783 7917 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.143.252:41589: Network error: Client connection negotiation failed: client connection to 127.7.143.252:41589: connect: Connection refused (error 111)
22:11:41.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:41.348155 7839 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.143.253" port: 45437 } attempt: 7
22:11:41.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:41.367273 7931 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:11:41.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 22:11:41 dist-test-slave-qzth krb5kdc[7755](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790547098, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.143.254@KRBTEST.COM for kudu/127.7.143.253@KRBTEST.COM
22:11:41.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:41.390111 7839 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.143.252" port: 41589 } has no permanent_uuid. Determining permanent_uuid...
22:11:41.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:41.395798 7839 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.143.252:41589: Network error: Client connection negotiation failed: client connection to 127.7.143.252:41589: connect: Connection refused (error 111)
22:11:41.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:41.454177 7839 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.143.252" port: 41589 } attempt: 1
22:11:41.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:41.458422 7839 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.143.252:41589: Network error: Client connection negotiation failed: client connection to 127.7.143.252:41589: connect: Connection refused (error 111)
22:11:41.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:41.539808 7839 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.143.252" port: 41589 } attempt: 2
22:11:41.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:41.544610 7839 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.143.252:41589: Network error: Client connection negotiation failed: client connection to 127.7.143.252:41589: connect: Connection refused (error 111)
22:11:41.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:41.629971 7839 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.143.252" port: 41589 } attempt: 3
22:11:41.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:41.634289 7839 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.143.252:41589: Network error: Client connection negotiation failed: client connection to 127.7.143.252:41589: connect: Connection refused (error 111)
22:11:41.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:41.647130 7917 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.143.252" port: 41589 } attempt: 5
22:11:41.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:41.650956 7917 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.143.252:41589: Network error: Client connection negotiation failed: client connection to 127.7.143.252:41589: connect: Connection refused (error 111)
22:11:41.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:41.774652 7839 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.143.252" port: 41589 } attempt: 4
22:11:41.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:41.778771 7839 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.143.252:41589: Network error: Client connection negotiation failed: client connection to 127.7.143.252:41589: connect: Connection refused (error 111)
22:11:41.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:41.968196 7248 debug-util.cc:398] Leaking SignalData structure 0x7b08000c2200 after lost signal to thread 7180
22:11:41.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:41.968658 7248 debug-util.cc:398] Leaking SignalData structure 0x7b080009d100 after lost signal to thread 7251
22:11:42.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:42.062203 7839 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.143.252" port: 41589 } attempt: 5
22:11:42.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:42.079538 7839 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.143.252:41589: Network error: Client connection negotiation failed: client connection to 127.7.143.252:41589: connect: Connection refused (error 111)
22:11:42.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:42.198377 7917 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.143.252" port: 41589 } attempt: 6
22:11:42.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:42.207306 7917 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.143.252:41589: Network error: Client connection negotiation failed: client connection to 127.7.143.252:41589: connect: Connection refused (error 111)
22:11:42.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:42.631974 7839 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.143.252" port: 41589 } attempt: 6
22:11:42.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:42.638811 7839 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.143.252:41589: Network error: Client connection negotiation failed: client connection to 127.7.143.252:41589: connect: Connection refused (error 111)
22:11:42.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:41.368734 7935 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:11:42.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:41.367759 7933 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:11:42.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:42.677716 7923 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
22:11:42.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:42.679636 7934 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
22:11:42.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:42.682143 7923 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
22:11:42.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:42.685293 7923 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
22:11:42.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:42.686851 7923 hybrid_clock.cc:648] HybridClock initialized: now 1790547102686783 us; error 44 us; skew 500 ppm
22:11:42.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:42.687943 7923 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
22:11:42.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 22:11:42 dist-test-slave-qzth krb5kdc[7755](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790547102, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.143.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
22:11:42.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:42.694824 7923 init.cc:377] Logged in from keytab as kudu/127.7.143.252@KRBTEST.COM (short username kudu)
22:11:42.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:42.700006 7923 webserver.cc:460] Webserver started at http://127.7.143.252:46595/ using document root <none> and password file <none>
22:11:42.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:42.701023 7923 fs_manager.cc:359] Metadata directory not provided
22:11:42.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:42.701265 7923 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
22:11:42.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:42.701730 7923 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
22:11:42.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:42.706444 7923 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-2/data/instance:
22:11:42.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8e8c40d6791a43a99f67354e2df309ee"
22:11:42.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 22:11:42 on dist-test-slave-qzth"
22:11:42.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:42.707448 7923 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-2/wal/instance:
22:11:42.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8e8c40d6791a43a99f67354e2df309ee"
22:11:42.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 22:11:42 on dist-test-slave-qzth"
22:11:42.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:42.715188 7923 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.008s sys 0.000s
22:11:42.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:42.721206 7951 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
22:11:42.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:42.722534 7923 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.000s
22:11:42.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:42.722806 7923 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-2/data,/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-2/wal
22:11:42.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8e8c40d6791a43a99f67354e2df309ee"
22:11:42.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 22:11:42 on dist-test-slave-qzth"
22:11:42.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:42.723073 7923 fs_report.cc:383] FS layout report
22:11:42.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
22:11:42.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-2/wal
22:11:42.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-2/wal
22:11:42.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-2/data/data
22:11:42.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
22:11:42.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
22:11:42.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
22:11:42.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
22:11:42.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
22:11:42.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
22:11:42.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
22:11:42.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
22:11:42.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
22:11:42.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
22:11:42.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
22:11:42.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:42.771512 7923 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
22:11:42.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:42.777705 7923 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
22:11:42.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:42.778144 7923 kserver.cc:163] Server-wide thread pool size limit: 3276
22:11:42.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:42.868288 7923 rpc_server.cc:307] RPC server started. Bound to: 127.7.143.252:41589
22:11:42.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:42.868357 8002 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.143.252:41589 every 8 connection(s)
22:11:42.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:42.871012 7923 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-2/data/info.pb
22:11:42.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:42.875408 7743 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu as pid 7923
22:11:42.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:42.875988 7743 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-2/wal/instance
22:11:42.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:42.884783 8003 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
22:11:42.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:42.915167 8003 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.143.254" port: 40351 } has no permanent_uuid. Determining permanent_uuid...
22:11:42.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 22:11:42 dist-test-slave-qzth krb5kdc[7755](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790547102, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.143.252@KRBTEST.COM for kudu/127.7.143.254@KRBTEST.COM
22:11:42.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 22:11:42 dist-test-slave-qzth krb5kdc[7755](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790547097, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.7.143.252@KRBTEST.COM
22:11:42.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:42.968501 8003 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.143.253" port: 45437 } has no permanent_uuid. Determining permanent_uuid...
22:11:42.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 22:11:42 dist-test-slave-qzth krb5kdc[7755](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790547102, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.143.252@KRBTEST.COM for kudu/127.7.143.253@KRBTEST.COM
22:11:42.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 22:11:42 dist-test-slave-qzth krb5kdc[7755](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790547097, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.7.143.253@KRBTEST.COM
22:11:43.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:43.007929 8003 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.143.252" port: 41589 } has no permanent_uuid. Determining permanent_uuid...
22:11:43.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 22:11:43 dist-test-slave-qzth krb5kdc[7755](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790547102, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.143.252@KRBTEST.COM for kudu/127.7.143.252@KRBTEST.COM
22:11:43.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 22:11:43 dist-test-slave-qzth krb5kdc[7755](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790547097, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.7.143.254@KRBTEST.COM
22:11:43.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:43.041946 8003 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee: Bootstrap starting.
22:11:43.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:43.044549 7743 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
22:11:43.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:43.048712 8003 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee: Neither blocks nor log segments found. Creating new log.
22:11:43.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:43.050606 8003 log.cc:824] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee: Log is configured to *not* fsync() on all Append() calls
22:11:43.055 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
22:11:43.056 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
22:11:43.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:43.056756 8003 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee: No bootstrap required, opened a new log
22:11:43.069 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
22:11:43.070 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
22:11:43.073 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.7.143.254"
port: 40351
}
bound_http_address {
host: "127.7.143.254"
port: 45673
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.7.143.253"
port: 45437
}
bound_http_address {
host: "127.7.143.253"
port: 35067
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.7.143.252"
port: 41589
}
bound_http_address {
host: "127.7.143.252"
port: 46595
}
}
}
22:11:43.073 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
22:11:43.074 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
22:11:43.074 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
22:11:43.075 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
22:11:43.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:43.108630 8003 raft_consensus.cc:348] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "929b15eec5ef4bd9adb23cb19aea80f2" member_type: VOTER last_known_addr { host: "127.7.143.254" port: 40351 } } peers { permanent_uuid: "65fbe4442b8b4d21af2fa7d6b162abb7" member_type: VOTER last_known_addr { host: "127.7.143.253" port: 45437 } } peers { permanent_uuid: "8e8c40d6791a43a99f67354e2df309ee" member_type: VOTER last_known_addr { host: "127.7.143.252" port: 41589 } }
22:11:43.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:43.110957 8003 raft_consensus.cc:374] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
22:11:43.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:43.111424 8003 raft_consensus.cc:729] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8e8c40d6791a43a99f67354e2df309ee, State: Initialized, Role: FOLLOWER
22:11:43.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:43.112607 8003 consensus_queue.cc:260] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "929b15eec5ef4bd9adb23cb19aea80f2" member_type: VOTER last_known_addr { host: "127.7.143.254" port: 40351 } } peers { permanent_uuid: "65fbe4442b8b4d21af2fa7d6b162abb7" member_type: VOTER last_known_addr { host: "127.7.143.253" port: 45437 } } peers { permanent_uuid: "8e8c40d6791a43a99f67354e2df309ee" member_type: VOTER last_known_addr { host: "127.7.143.252" port: 41589 } }
22:11:43.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 22:11:43 dist-test-slave-qzth krb5kdc[7755](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790547103, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
22:11:43.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
22:11:43.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:43.118921 8015 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "929b15eec5ef4bd9adb23cb19aea80f2" member_type: VOTER last_known_addr { host: "127.7.143.254" port: 40351 } } peers { permanent_uuid: "65fbe4442b8b4d21af2fa7d6b162abb7" member_type: VOTER last_known_addr { host: "127.7.143.253" port: 45437 } } peers { permanent_uuid: "8e8c40d6791a43a99f67354e2df309ee" member_type: VOTER last_known_addr { host: "127.7.143.252" port: 41589 } } }
22:11:43.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:43.119814 8015 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee [sys.catalog]: This master's current role is: FOLLOWER
22:11:43.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:43.120754 8003 sys_catalog.cc:564] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee [sys.catalog]: configured and running, proceeding with master startup.
22:11:43.123 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
22:11:43.126 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/krb5kdc/krb5cc
22:11:43.135 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
22:11:43.136 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/krb5kdc/krb5cc
22:11:43.138 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
22:11:43.143 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
22:11:43.145 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
22:11:43.145 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
22:11:43.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:43.176391 8027 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:11:43.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 22:11:43 dist-test-slave-qzth krb5kdc[7755](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790547103, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
22:11:43.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
22:11:43.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:43.190124 8027 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:11:43.193 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
22:11:43.193 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/krb5kdc/krb5cc
22:11:43.197 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
22:11:43.197 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.7.143.252:41589
22:11:43.199 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
22:11:43.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:43.199985 7743 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu with pid 7923
22:11:43.222 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
22:11:43.223 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.7.143.254:40351
22:11:43.223 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
22:11:43.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:43.223753 7743 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu with pid 7769
22:11:43.256 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
22:11:43.256 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.7.143.253:45437
22:11:43.256 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
22:11:43.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:43.257129 7743 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu with pid 7845
22:11:43.279 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
22:11:43.279 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.7.143.252:41589
22:11:43.283 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
22:11:43.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:43.284951 7743 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu
22:11:43.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu
22:11:43.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-2/wal
22:11:43.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-2/data
22:11:43.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
22:11:43.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
22:11:43.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
22:11:43.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:11:43.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:11:43.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:11:43.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-2/logs
22:11:43.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-2/data/info.pb
22:11:43.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:11:43.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
22:11:43.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
22:11:43.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
22:11:43.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
22:11:43.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:11:43.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:11:43.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:11:43.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
22:11:43.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
22:11:43.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
22:11:43.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
22:11:43.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.143.252:41589
22:11:43.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.143.252
22:11:43.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46595
22:11:43.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/krb5kdc/kudu_127.7.143.252.keytab
22:11:43.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.143.252
22:11:43.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
22:11:43.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
22:11:43.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
22:11:43.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.143.212:44837
22:11:43.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:11:43.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:11:43.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:11:43.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
22:11:43.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.143.254:40351,127.7.143.253:45437,127.7.143.252:41589
22:11:43.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
22:11:43.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34207/jwks.json
22:11:43.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
22:11:43.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:43.624172 8030 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
22:11:43.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:43.624847 8030 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
22:11:43.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:43.625548 8030 flags.cc:425] Enabled unsafe flag: --never_fsync=true
22:11:43.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:43.656837 8030 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
22:11:43.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:43.657099 8030 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
22:11:43.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:43.657509 8030 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
22:11:43.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:43.657868 8030 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
22:11:43.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:43.658192 8030 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
22:11:43.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:43.658527 8030 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
22:11:43.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:43.658767 8030 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34207/jwks.json
22:11:43.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:43.701490 8030 master_runner.cc:386] Master server non-default flags:
22:11:43.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:11:43.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.143.212:44837
22:11:43.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:11:43.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:11:43.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-2/data
22:11:43.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-2/wal
22:11:43.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
22:11:43.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
22:11:43.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.143.254:40351,127.7.143.253:45437,127.7.143.252:41589
22:11:43.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/krb5kdc/kudu_127.7.143.252.keytab
22:11:43.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.143.252
22:11:43.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:11:43.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:11:43.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
22:11:43.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.143.252:41589
22:11:43.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
22:11:43.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
22:11:43.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
22:11:43.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34207/jwks.json
22:11:43.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
22:11:43.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
22:11:43.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
22:11:43.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:11:43.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:11:43.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-2/data/info.pb
22:11:43.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.143.252
22:11:43.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46595
22:11:43.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
22:11:43.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:11:43.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
22:11:43.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
22:11:43.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:11:43.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-2/logs
22:11:43.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:11:43.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
22:11:43.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:11:43.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
22:11:43.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
22:11:43.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
22:11:43.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
22:11:43.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 21:22:24 UTC on ccycloud.quasar-wysbne.root.comops.site
22:11:43.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676318
22:11:43.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
22:11:43.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:43.702885 8030 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
22:11:43.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:43.704205 8030 file_cache.cc:493] Constructed file cache file cache with capacity 419430
22:11:43.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:43.725363 8038 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:11:44.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:43.727269 8037 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:11:44.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:43.728786 8040 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:11:44.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:44.989480 8039 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
22:11:44.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:44.989606 8030 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
22:11:44.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:44.993413 8030 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
22:11:44.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:44.996584 8030 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
22:11:44.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:44.997979 8030 hybrid_clock.cc:648] HybridClock initialized: now 1790547104997940 us; error 60 us; skew 500 ppm
22:11:44.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:44.998752 8030 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
22:11:45.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 22:11:45 dist-test-slave-qzth krb5kdc[7755](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790547105, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.143.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
22:11:45.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:45.004385 8030 init.cc:377] Logged in from keytab as kudu/127.7.143.252@KRBTEST.COM (short username kudu)
22:11:45.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:45.011106 8030 webserver.cc:460] Webserver started at http://127.7.143.252:46595/ using document root <none> and password file <none>
22:11:45.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:45.012077 8030 fs_manager.cc:359] Metadata directory not provided
22:11:45.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:45.012272 8030 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
22:11:45.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:45.019964 8030 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.001s sys 0.004s
22:11:45.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:45.024194 8049 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
22:11:45.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:45.025297 8030 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
22:11:45.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:45.025664 8030 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-2/data,/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-2/wal
22:11:45.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8e8c40d6791a43a99f67354e2df309ee"
22:11:45.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 22:11:42 on dist-test-slave-qzth"
22:11:45.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:45.027974 8030 fs_report.cc:383] FS layout report
22:11:45.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
22:11:45.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-2/wal
22:11:45.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-2/wal
22:11:45.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-2/data/data
22:11:45.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
22:11:45.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
22:11:45.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
22:11:45.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
22:11:45.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
22:11:45.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
22:11:45.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
22:11:45.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
22:11:45.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
22:11:45.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
22:11:45.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
22:11:45.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:45.066685 8030 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
22:11:45.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:45.072844 8030 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
22:11:45.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:45.073436 8030 kserver.cc:163] Server-wide thread pool size limit: 3276
22:11:45.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:45.145628 8030 rpc_server.cc:307] RPC server started. Bound to: 127.7.143.252:41589
22:11:45.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:45.145725 8100 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.143.252:41589 every 8 connection(s)
22:11:45.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:45.148793 8030 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-2/data/info.pb
22:11:45.152 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
22:11:45.152 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.7.143.254:40351
22:11:45.152 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
22:11:45.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:45.151058 7743 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu as pid 8030
22:11:45.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:45.154207 7743 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu
22:11:45.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu
22:11:45.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-0/wal
22:11:45.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-0/data
22:11:45.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
22:11:45.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
22:11:45.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
22:11:45.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:11:45.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:11:45.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:11:45.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-0/logs
22:11:45.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-0/data/info.pb
22:11:45.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:11:45.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
22:11:45.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
22:11:45.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
22:11:45.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
22:11:45.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:11:45.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:11:45.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:11:45.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
22:11:45.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
22:11:45.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
22:11:45.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
22:11:45.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.143.254:40351
22:11:45.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.143.254
22:11:45.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45673
22:11:45.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/krb5kdc/kudu_127.7.143.254.keytab
22:11:45.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.143.254
22:11:45.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
22:11:45.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
22:11:45.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
22:11:45.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.143.212:44837
22:11:45.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:11:45.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:11:45.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:11:45.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
22:11:45.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.143.254:40351,127.7.143.253:45437,127.7.143.252:41589
22:11:45.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
22:11:45.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34207/jwks.json
22:11:45.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
22:11:45.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:45.162135 8101 sys_catalog.cc:263] Verifying existing consensus state
22:11:45.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:45.168048 8101 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee: Bootstrap starting.
22:11:45.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:45.200256 8101 log.cc:824] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee: Log is configured to *not* fsync() on all Append() calls
22:11:45.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:45.205351 8101 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
22:11:45.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:45.206112 8101 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee: Bootstrap complete.
22:11:45.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:45.235447 8101 raft_consensus.cc:348] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "929b15eec5ef4bd9adb23cb19aea80f2" member_type: VOTER last_known_addr { host: "127.7.143.254" port: 40351 } } peers { permanent_uuid: "65fbe4442b8b4d21af2fa7d6b162abb7" member_type: VOTER last_known_addr { host: "127.7.143.253" port: 45437 } } peers { permanent_uuid: "8e8c40d6791a43a99f67354e2df309ee" member_type: VOTER last_known_addr { host: "127.7.143.252" port: 41589 } }
22:11:45.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:45.237730 8101 raft_consensus.cc:374] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
22:11:45.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:45.238082 8101 raft_consensus.cc:729] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8e8c40d6791a43a99f67354e2df309ee, State: Initialized, Role: FOLLOWER
22:11:45.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:45.238968 8101 consensus_queue.cc:260] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "929b15eec5ef4bd9adb23cb19aea80f2" member_type: VOTER last_known_addr { host: "127.7.143.254" port: 40351 } } peers { permanent_uuid: "65fbe4442b8b4d21af2fa7d6b162abb7" member_type: VOTER last_known_addr { host: "127.7.143.253" port: 45437 } } peers { permanent_uuid: "8e8c40d6791a43a99f67354e2df309ee" member_type: VOTER last_known_addr { host: "127.7.143.252" port: 41589 } }
22:11:45.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:45.241803 8106 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "929b15eec5ef4bd9adb23cb19aea80f2" member_type: VOTER last_known_addr { host: "127.7.143.254" port: 40351 } } peers { permanent_uuid: "65fbe4442b8b4d21af2fa7d6b162abb7" member_type: VOTER last_known_addr { host: "127.7.143.253" port: 45437 } } peers { permanent_uuid: "8e8c40d6791a43a99f67354e2df309ee" member_type: VOTER last_known_addr { host: "127.7.143.252" port: 41589 } } }
22:11:45.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:45.242638 8106 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee [sys.catalog]: This master's current role is: FOLLOWER
22:11:45.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:45.244798 8101 sys_catalog.cc:564] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee [sys.catalog]: configured and running, proceeding with master startup.
22:11:45.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:45.282105 8117 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:11:45.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:45.282665 8117 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:11:45.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:45.273922 8030 master_runner.cc:186] Error getting master registration for 127.7.143.254:40351: Network error: Client connection negotiation failed: client connection to 127.7.143.254:40351: connect: Connection refused (error 111),
22:11:45.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:45.295354 8030 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
22:11:45.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:45.328964 8055 tablet.cc:2369] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee: Can't schedule compaction. Clean time has not been advanced past its initial value.
22:11:45.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:45.512101 8106 raft_consensus.cc:482] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
22:11:45.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:45.512653 8106 raft_consensus.cc:504] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "929b15eec5ef4bd9adb23cb19aea80f2" member_type: VOTER last_known_addr { host: "127.7.143.254" port: 40351 } } peers { permanent_uuid: "65fbe4442b8b4d21af2fa7d6b162abb7" member_type: VOTER last_known_addr { host: "127.7.143.253" port: 45437 } } peers { permanent_uuid: "8e8c40d6791a43a99f67354e2df309ee" member_type: VOTER last_known_addr { host: "127.7.143.252" port: 41589 } }
22:11:45.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:45.518105 8106 leader_election.cc:288] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 929b15eec5ef4bd9adb23cb19aea80f2 (127.7.143.254:40351), 65fbe4442b8b4d21af2fa7d6b162abb7 (127.7.143.253:45437)
22:11:45.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:45.525107 8104 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
22:11:45.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:45.525282 8050 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.143.254:40351: connect: Connection refused (error 111)
22:11:45.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:45.525918 8104 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
22:11:45.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:45.527201 8104 flags.cc:425] Enabled unsafe flag: --never_fsync=true
22:11:45.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:45.532178 8050 leader_election.cc:334] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 929b15eec5ef4bd9adb23cb19aea80f2 (127.7.143.254:40351): Network error: Client connection negotiation failed: client connection to 127.7.143.254:40351: connect: Connection refused (error 111)
22:11:45.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:45.532828 8050 leader_election.cc:334] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 65fbe4442b8b4d21af2fa7d6b162abb7 (127.7.143.253:45437): Network error: Client connection negotiation failed: client connection to 127.7.143.253:45437: connect: Connection refused (error 111)
22:11:45.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:45.533164 8050 leader_election.cc:302] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 8e8c40d6791a43a99f67354e2df309ee; no voters: 65fbe4442b8b4d21af2fa7d6b162abb7, 929b15eec5ef4bd9adb23cb19aea80f2
22:11:45.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:45.533978 8106 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
22:11:45.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:45.564754 8104 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
22:11:45.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:45.564955 8104 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
22:11:45.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:45.565217 8104 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
22:11:45.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:45.565418 8104 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
22:11:45.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:45.565613 8104 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
22:11:45.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:45.565786 8104 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
22:11:45.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:45.566004 8104 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34207/jwks.json
22:11:45.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:45.601830 8104 master_runner.cc:386] Master server non-default flags:
22:11:45.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:11:45.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.143.212:44837
22:11:45.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:11:45.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:11:45.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-0/data
22:11:45.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-0/wal
22:11:45.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
22:11:45.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
22:11:45.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.143.254:40351,127.7.143.253:45437,127.7.143.252:41589
22:11:45.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/krb5kdc/kudu_127.7.143.254.keytab
22:11:45.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.143.254
22:11:45.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:11:45.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:11:45.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
22:11:45.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.143.254:40351
22:11:45.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
22:11:45.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
22:11:45.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
22:11:45.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34207/jwks.json
22:11:45.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
22:11:45.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
22:11:45.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
22:11:45.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:11:45.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:11:45.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-0/data/info.pb
22:11:45.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.143.254
22:11:45.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45673
22:11:45.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
22:11:45.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:11:45.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
22:11:45.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
22:11:45.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:11:45.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-0/logs
22:11:45.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:11:45.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
22:11:45.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:11:45.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
22:11:45.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
22:11:45.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
22:11:45.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
22:11:45.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 21:22:24 UTC on ccycloud.quasar-wysbne.root.comops.site
22:11:45.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676318
22:11:45.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
22:11:45.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:45.602948 8104 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
22:11:45.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:45.604228 8104 file_cache.cc:493] Constructed file cache file cache with capacity 419430
22:11:45.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:45.625967 8127 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:11:46.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:46.287178 8117 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:11:46.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:46.287493 8117 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:11:46.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:45.628644 8104 server_base.cc:1034] running on GCE node
22:11:46.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:45.627646 8128 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:11:46.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:45.626519 8130 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:11:46.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:46.900288 8104 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
22:11:46.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:46.903538 8104 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
22:11:46.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:46.905148 8104 hybrid_clock.cc:648] HybridClock initialized: now 1790547106905080 us; error 80 us; skew 500 ppm
22:11:46.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:46.906008 8104 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
22:11:46.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 22:11:46 dist-test-slave-qzth krb5kdc[7755](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790547106, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.143.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
22:11:46.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:46.917133 8104 init.cc:377] Logged in from keytab as kudu/127.7.143.254@KRBTEST.COM (short username kudu)
22:11:46.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:46.923334 8104 webserver.cc:460] Webserver started at http://127.7.143.254:45673/ using document root <none> and password file <none>
22:11:46.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:46.924216 8104 fs_manager.cc:359] Metadata directory not provided
22:11:46.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:46.924460 8104 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
22:11:46.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:46.933604 8104 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.006s sys 0.001s
22:11:46.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:46.938558 8140 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
22:11:46.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:46.939746 8104 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.001s sys 0.003s
22:11:46.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:46.940066 8104 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-0/data,/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-0/wal
22:11:46.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "929b15eec5ef4bd9adb23cb19aea80f2"
22:11:46.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 22:11:38 on dist-test-slave-qzth"
22:11:46.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:46.942517 8104 fs_report.cc:383] FS layout report
22:11:46.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
22:11:46.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-0/wal
22:11:46.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-0/wal
22:11:46.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-0/data/data
22:11:46.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
22:11:46.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
22:11:46.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
22:11:46.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
22:11:46.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
22:11:46.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
22:11:46.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
22:11:46.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
22:11:46.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
22:11:46.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
22:11:46.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
22:11:46.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:46.997620 8104 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
22:11:47.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:47.003638 8104 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
22:11:47.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:47.004150 8104 kserver.cc:163] Server-wide thread pool size limit: 3276
22:11:47.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:47.099979 8104 rpc_server.cc:307] RPC server started. Bound to: 127.7.143.254:40351
22:11:47.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:47.100099 8191 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.143.254:40351 every 8 connection(s)
22:11:47.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:47.103149 8104 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-0/data/info.pb
22:11:47.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:47.113534 7743 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu as pid 8104
22:11:47.115 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
22:11:47.115 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.7.143.253:45437
22:11:47.115 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
22:11:47.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:47.115749 8192 sys_catalog.cc:263] Verifying existing consensus state
22:11:47.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260927 22:11:47.116869 8192 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
22:11:47.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:47.117347 8104 master.cc:537] Master@127.7.143.254:40351 shutting down...
22:11:47.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:47.118548 7743 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu
22:11:47.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu
22:11:47.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-1/wal
22:11:47.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-1/data
22:11:47.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
22:11:47.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
22:11:47.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
22:11:47.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:11:47.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:11:47.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:11:47.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-1/logs
22:11:47.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-1/data/info.pb
22:11:47.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:11:47.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
22:11:47.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
22:11:47.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
22:11:47.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
22:11:47.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:11:47.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:11:47.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:11:47.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
22:11:47.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
22:11:47.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
22:11:47.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
22:11:47.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.143.253:45437
22:11:47.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.143.253
22:11:47.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35067
22:11:47.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/krb5kdc/kudu_127.7.143.253.keytab
22:11:47.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.143.253
22:11:47.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
22:11:47.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
22:11:47.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
22:11:47.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.143.212:44837
22:11:47.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:11:47.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:11:47.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:11:47.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
22:11:47.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.143.254:40351,127.7.143.253:45437,127.7.143.252:41589
22:11:47.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
22:11:47.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34207/jwks.json
22:11:47.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
22:11:47.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:47.192962 8104 master.cc:552] Master@127.7.143.254:40351 shutdown complete.
22:11:47.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
22:11:47.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:47.290496 8117 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:11:47.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:47.290735 8117 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:11:47.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:47.371021 8195 raft_consensus.cc:482] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
22:11:47.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:47.371342 8195 raft_consensus.cc:504] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "929b15eec5ef4bd9adb23cb19aea80f2" member_type: VOTER last_known_addr { host: "127.7.143.254" port: 40351 } } peers { permanent_uuid: "65fbe4442b8b4d21af2fa7d6b162abb7" member_type: VOTER last_known_addr { host: "127.7.143.253" port: 45437 } } peers { permanent_uuid: "8e8c40d6791a43a99f67354e2df309ee" member_type: VOTER last_known_addr { host: "127.7.143.252" port: 41589 } }
22:11:47.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:47.372977 8195 leader_election.cc:288] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 929b15eec5ef4bd9adb23cb19aea80f2 (127.7.143.254:40351), 65fbe4442b8b4d21af2fa7d6b162abb7 (127.7.143.253:45437)
22:11:47.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:47.380322 8050 leader_election.cc:334] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 929b15eec5ef4bd9adb23cb19aea80f2 (127.7.143.254:40351): Network error: Client connection negotiation failed: client connection to 127.7.143.254:40351: connect: Connection refused (error 111)
22:11:47.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:47.380848 8050 leader_election.cc:334] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 65fbe4442b8b4d21af2fa7d6b162abb7 (127.7.143.253:45437): Network error: Client connection negotiation failed: client connection to 127.7.143.253:45437: connect: Connection refused (error 111)
22:11:47.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:47.381145 8050 leader_election.cc:302] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 8e8c40d6791a43a99f67354e2df309ee; no voters: 65fbe4442b8b4d21af2fa7d6b162abb7, 929b15eec5ef4bd9adb23cb19aea80f2
22:11:47.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:47.381758 8195 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
22:11:47.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:47.450419 8194 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
22:11:47.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:47.451092 8194 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
22:11:47.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:47.451756 8194 flags.cc:425] Enabled unsafe flag: --never_fsync=true
22:11:47.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:47.484207 8194 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
22:11:47.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:47.484467 8194 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
22:11:47.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:47.484915 8194 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
22:11:47.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:47.485270 8194 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
22:11:47.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:47.485574 8194 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
22:11:47.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:47.485916 8194 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
22:11:47.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:47.486141 8194 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34207/jwks.json
22:11:47.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:47.522672 8194 master_runner.cc:386] Master server non-default flags:
22:11:47.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:11:47.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.143.212:44837
22:11:47.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:11:47.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:11:47.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-1/data
22:11:47.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-1/wal
22:11:47.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
22:11:47.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
22:11:47.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.143.254:40351,127.7.143.253:45437,127.7.143.252:41589
22:11:47.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/krb5kdc/kudu_127.7.143.253.keytab
22:11:47.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.143.253
22:11:47.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:11:47.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:11:47.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
22:11:47.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.143.253:45437
22:11:47.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
22:11:47.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
22:11:47.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
22:11:47.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34207/jwks.json
22:11:47.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
22:11:47.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
22:11:47.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
22:11:47.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:11:47.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:11:47.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-1/data/info.pb
22:11:47.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.143.253
22:11:47.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35067
22:11:47.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
22:11:47.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:11:47.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
22:11:47.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
22:11:47.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:11:47.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-1/logs
22:11:47.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:11:47.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
22:11:47.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:11:47.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
22:11:47.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
22:11:47.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
22:11:47.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
22:11:47.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 21:22:24 UTC on ccycloud.quasar-wysbne.root.comops.site
22:11:47.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676318
22:11:47.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
22:11:47.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:47.524082 8194 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
22:11:47.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:47.525404 8194 file_cache.cc:493] Constructed file cache file cache with capacity 419430
22:11:47.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:47.542827 8204 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:11:48.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:48.293550 8117 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:11:48.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:48.293845 8117 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:11:48.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:47.544001 8207 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:11:48.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:47.545331 8194 server_base.cc:1034] running on GCE node
22:11:48.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:47.542862 8205 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:11:48.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:48.751029 8194 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
22:11:48.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:48.754016 8194 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
22:11:48.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:48.755416 8194 hybrid_clock.cc:648] HybridClock initialized: now 1790547108755387 us; error 51 us; skew 500 ppm
22:11:48.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:48.756354 8194 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
22:11:48.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 22:11:48 dist-test-slave-qzth krb5kdc[7755](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790547108, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.143.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
22:11:48.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:48.762477 8194 init.cc:377] Logged in from keytab as kudu/127.7.143.253@KRBTEST.COM (short username kudu)
22:11:48.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:48.768538 8194 webserver.cc:460] Webserver started at http://127.7.143.253:35067/ using document root <none> and password file <none>
22:11:48.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:48.769356 8194 fs_manager.cc:359] Metadata directory not provided
22:11:48.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:48.769541 8194 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
22:11:48.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:48.777110 8194 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.006s sys 0.000s
22:11:48.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:48.781064 8216 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
22:11:48.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:48.782097 8194 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
22:11:48.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:48.782341 8194 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-1/data,/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-1/wal
22:11:48.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "65fbe4442b8b4d21af2fa7d6b162abb7"
22:11:48.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 22:11:40 on dist-test-slave-qzth"
22:11:48.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:48.784461 8194 fs_report.cc:383] FS layout report
22:11:48.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
22:11:48.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-1/wal
22:11:48.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-1/wal
22:11:48.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-1/data/data
22:11:48.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
22:11:48.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
22:11:48.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
22:11:48.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
22:11:48.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
22:11:48.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
22:11:48.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
22:11:48.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
22:11:48.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
22:11:48.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
22:11:48.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
22:11:48.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:48.830245 8194 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
22:11:48.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:48.835718 8194 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
22:11:48.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:48.836097 8194 kserver.cc:163] Server-wide thread pool size limit: 3276
22:11:48.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:48.904325 8194 rpc_server.cc:307] RPC server started. Bound to: 127.7.143.253:45437
22:11:48.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:48.904561 8267 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.143.253:45437 every 8 connection(s)
22:11:48.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:48.907189 8194 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-1/data/info.pb
22:11:48.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:48.912236 7743 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu as pid 8194
22:11:48.912 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
22:11:48.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:48.917688 8268 sys_catalog.cc:263] Verifying existing consensus state
22:11:48.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260927 22:11:48.918614 8268 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
22:11:48.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:48.919018 8194 master.cc:537] Master@127.7.143.253:45437 shutting down...
22:11:48.963 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.7.143.252:41589(127.7.143.252:41589)] connecting to peer
22:11:48.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:48.980994 8194 master.cc:552] Master@127.7.143.253:45437 shutdown complete.
22:11:48.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster663868791387549172/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
22:11:49.027 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.7.143.254:40351(127.7.143.254:40351)] connecting to peer
22:11:49.028 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.7.143.253:45437(127.7.143.253:45437)] connecting to peer
22:11:49.084 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.7.143.252:41589(127.7.143.252:41589)] Successfully connected to peer
22:11:49.086 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:11:49.087 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.143.254:40351(127.7.143.254:40351)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:11:49.089 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.143.254:40351: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:11:49.137 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:11:49.137 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.143.253:45437(127.7.143.253:45437)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:11:49.138 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.143.253:45437: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:11:49.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:49.150273 8274 raft_consensus.cc:482] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
22:11:49.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:49.150969 8274 raft_consensus.cc:504] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "929b15eec5ef4bd9adb23cb19aea80f2" member_type: VOTER last_known_addr { host: "127.7.143.254" port: 40351 } } peers { permanent_uuid: "65fbe4442b8b4d21af2fa7d6b162abb7" member_type: VOTER last_known_addr { host: "127.7.143.253" port: 45437 } } peers { permanent_uuid: "8e8c40d6791a43a99f67354e2df309ee" member_type: VOTER last_known_addr { host: "127.7.143.252" port: 41589 } }
22:11:49.157 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
22:11:49.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:49.159092 8274 leader_election.cc:288] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 929b15eec5ef4bd9adb23cb19aea80f2 (127.7.143.254:40351), 65fbe4442b8b4d21af2fa7d6b162abb7 (127.7.143.253:45437)
22:11:49.168 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.7.143.252
22:11:49.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:49.169433 8050 leader_election.cc:334] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 929b15eec5ef4bd9adb23cb19aea80f2 (127.7.143.254:40351): Network error: Client connection negotiation failed: client connection to 127.7.143.254:40351: connect: Connection refused (error 111)
22:11:49.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:49.170217 8050 leader_election.cc:334] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 65fbe4442b8b4d21af2fa7d6b162abb7 (127.7.143.253:45437): Network error: Client connection negotiation failed: client connection to 127.7.143.253:45437: connect: Connection refused (error 111)
22:11:49.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:49.170591 8050 leader_election.cc:302] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 8e8c40d6791a43a99f67354e2df309ee; no voters: 65fbe4442b8b4d21af2fa7d6b162abb7, 929b15eec5ef4bd9adb23cb19aea80f2
22:11:49.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:49.171300 8274 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
22:11:49.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:49.297711 8117 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:11:49.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:49.298646 8117 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:11:49.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 22:11:49 dist-test-slave-qzth krb5kdc[7755](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790547103, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.7.143.252@KRBTEST.COM
22:11:49.390 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x0aa04498, L:/127.0.0.1:32840 - R:/127.7.143.252:41589] using SASL/GSSAPI
22:11:49.403 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.143.252:41589,127.7.143.254:40351,127.7.143.253:45437; will retry
22:11:49.419 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.143.254:40351(127.7.143.254:40351)] connecting to peer
22:11:49.422 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.143.253:45437(127.7.143.253:45437)] connecting to peer
22:11:49.422 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:11:49.422 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.143.254:40351(127.7.143.254:40351)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:11:49.423 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.143.254:40351: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:11:49.425 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:11:49.425 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.143.253:45437(127.7.143.253:45437)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:11:49.426 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.143.253:45437: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:11:49.426 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.143.252:41589,127.7.143.254:40351,127.7.143.253:45437; will retry
22:11:49.437 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.143.254:40351(127.7.143.254:40351)] connecting to peer
22:11:49.438 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.143.253:45437(127.7.143.253:45437)] connecting to peer
22:11:49.439 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:11:49.440 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.143.254:40351(127.7.143.254:40351)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:11:49.440 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.143.254:40351: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:11:49.442 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:11:49.442 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.143.253:45437(127.7.143.253:45437)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:11:49.443 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.143.253:45437: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:11:49.444 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.143.252:41589,127.7.143.254:40351,127.7.143.253:45437; will retry
22:11:49.457 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.143.254:40351(127.7.143.254:40351)] connecting to peer
22:11:49.458 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.143.253:45437(127.7.143.253:45437)] connecting to peer
22:11:49.460 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:11:49.461 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.143.254:40351(127.7.143.254:40351)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:11:49.461 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.143.254:40351: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:11:49.462 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:11:49.462 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.143.253:45437(127.7.143.253:45437)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:11:49.463 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.143.253:45437: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:11:49.463 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.143.252:41589,127.7.143.254:40351,127.7.143.253:45437; will retry
22:11:49.477 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.143.254:40351(127.7.143.254:40351)] connecting to peer
22:11:49.478 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.143.253:45437(127.7.143.253:45437)] connecting to peer
22:11:49.480 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:11:49.480 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.143.254:40351(127.7.143.254:40351)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:11:49.481 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.143.254:40351: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:11:49.482 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:11:49.482 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.143.253:45437(127.7.143.253:45437)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:11:49.482 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.143.253:45437: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:11:49.483 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.143.252:41589,127.7.143.254:40351,127.7.143.253:45437; will retry
22:11:49.497 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.143.254:40351(127.7.143.254:40351)] connecting to peer
22:11:49.498 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.143.253:45437(127.7.143.253:45437)] connecting to peer
22:11:49.500 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:11:49.500 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.143.254:40351(127.7.143.254:40351)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:11:49.501 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:11:49.502 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.143.253:45437(127.7.143.253:45437)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:11:49.502 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.143.254:40351: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:11:49.502 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.143.253:45437: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:11:49.503 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.143.252:41589,127.7.143.254:40351,127.7.143.253:45437; will retry
22:11:49.517 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.143.254:40351(127.7.143.254:40351)] connecting to peer
22:11:49.518 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.143.253:45437(127.7.143.253:45437)] connecting to peer
22:11:49.519 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:11:49.520 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.143.254:40351(127.7.143.254:40351)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:11:49.520 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.143.254:40351: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:11:49.522 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:11:49.522 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.143.253:45437(127.7.143.253:45437)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:11:49.523 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.143.253:45437: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:11:49.523 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.143.252:41589,127.7.143.254:40351,127.7.143.253:45437; will retry
22:11:49.638 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.143.254:40351(127.7.143.254:40351)] connecting to peer
22:11:49.640 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.143.253:45437(127.7.143.253:45437)] connecting to peer
22:11:49.640 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:11:49.640 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.143.254:40351(127.7.143.254:40351)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:11:49.641 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.143.254:40351: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:11:49.642 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:11:49.643 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.143.253:45437(127.7.143.253:45437)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:11:49.643 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.143.253:45437: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:11:49.644 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.143.252:41589,127.7.143.254:40351,127.7.143.253:45437; will retry
22:11:49.658 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.143.254:40351(127.7.143.254:40351)] connecting to peer
22:11:49.659 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.143.253:45437(127.7.143.253:45437)] connecting to peer
22:11:49.660 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:11:49.660 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.143.254:40351(127.7.143.254:40351)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:11:49.662 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.143.254:40351: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:11:49.662 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:11:49.662 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.143.253:45437(127.7.143.253:45437)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:11:49.662 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.143.253:45437: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:11:49.663 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.143.252:41589,127.7.143.254:40351,127.7.143.253:45437; will retry
22:11:49.818 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.143.254:40351(127.7.143.254:40351)] connecting to peer
22:11:49.819 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.143.253:45437(127.7.143.253:45437)] connecting to peer
22:11:49.821 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:11:49.821 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.143.254:40351(127.7.143.254:40351)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:11:49.822 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.143.254:40351: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:11:49.822 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:11:49.822 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.143.253:45437(127.7.143.253:45437)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:11:49.823 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.143.253:45437: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:11:49.823 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.143.252:41589,127.7.143.254:40351,127.7.143.253:45437; will retry
22:11:50.037 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.143.254:40351(127.7.143.254:40351)] connecting to peer
22:11:50.039 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.143.253:45437(127.7.143.253:45437)] connecting to peer
22:11:50.040 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:11:50.040 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.143.254:40351(127.7.143.254:40351)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:11:50.041 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.143.254:40351: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:11:50.042 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:11:50.042 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.143.253:45437(127.7.143.253:45437)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:11:50.043 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.143.253:45437: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:11:50.043 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.143.252:41589,127.7.143.254:40351,127.7.143.253:45437; will retry
22:11:50.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:50.297835 8030 master_runner.cc:186] Error getting master registration for 127.7.143.254:40351: Network error: Client connection negotiation failed: client connection to 127.7.143.254:40351: connect: Connection refused (error 111),
22:11:50.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:50.298233 8030 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
22:11:50.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:50.301316 8117 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:11:50.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:50.301543 8117 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:11:50.837 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.143.254:40351(127.7.143.254:40351)] connecting to peer
22:11:50.839 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.143.253:45437(127.7.143.253:45437)] connecting to peer
22:11:50.839 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:11:50.839 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.143.254:40351(127.7.143.254:40351)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:11:50.840 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.143.254:40351: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:11:50.840 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:11:50.840 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.143.253:45437(127.7.143.253:45437)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:11:50.840 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.143.253:45437: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:11:50.841 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.143.252:41589,127.7.143.254:40351,127.7.143.253:45437; will retry
22:11:50.978 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.143.254:40351(127.7.143.254:40351)] connecting to peer
22:11:50.979 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.143.253:45437(127.7.143.253:45437)] connecting to peer
22:11:50.980 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:11:50.980 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.143.254:40351(127.7.143.254:40351)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:11:50.981 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.143.254:40351: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:11:50.982 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:11:50.982 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.143.253:45437(127.7.143.253:45437)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:11:50.983 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.143.253:45437: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:11:50.983 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.143.252:41589,127.7.143.254:40351,127.7.143.253:45437; will retry
22:11:51.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:51.304490 8117 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:11:51.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:51.304762 8117 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:11:52.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:52.307489 8117 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:11:52.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:52.307777 8117 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:11:52.658 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.143.254:40351(127.7.143.254:40351)] connecting to peer
22:11:52.659 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.143.253:45437(127.7.143.253:45437)] connecting to peer
22:11:52.660 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:11:52.660 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.143.254:40351(127.7.143.254:40351)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:11:52.661 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.143.254:40351: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:11:52.662 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:11:52.662 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.143.253:45437(127.7.143.253:45437)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:11:52.662 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.143.253:45437: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:11:52.662 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.143.252:41589,127.7.143.254:40351,127.7.143.253:45437; will retry
22:11:53.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:53.310545 8117 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:11:53.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:53.310820 8117 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:11:54.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:54.201642 8284 raft_consensus.cc:482] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
22:11:54.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:54.202320 8284 raft_consensus.cc:504] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "929b15eec5ef4bd9adb23cb19aea80f2" member_type: VOTER last_known_addr { host: "127.7.143.254" port: 40351 } } peers { permanent_uuid: "65fbe4442b8b4d21af2fa7d6b162abb7" member_type: VOTER last_known_addr { host: "127.7.143.253" port: 45437 } } peers { permanent_uuid: "8e8c40d6791a43a99f67354e2df309ee" member_type: VOTER last_known_addr { host: "127.7.143.252" port: 41589 } }
22:11:54.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:54.204334 8284 leader_election.cc:288] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 929b15eec5ef4bd9adb23cb19aea80f2 (127.7.143.254:40351), 65fbe4442b8b4d21af2fa7d6b162abb7 (127.7.143.253:45437)
22:11:54.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:54.207960 8050 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.143.254:40351: connect: Connection refused (error 111) [suppressed 5 similar messages]
22:11:54.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:54.212199 8050 leader_election.cc:334] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 929b15eec5ef4bd9adb23cb19aea80f2 (127.7.143.254:40351): Network error: Client connection negotiation failed: client connection to 127.7.143.254:40351: connect: Connection refused (error 111)
22:11:54.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:54.212756 8050 leader_election.cc:334] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 65fbe4442b8b4d21af2fa7d6b162abb7 (127.7.143.253:45437): Network error: Client connection negotiation failed: client connection to 127.7.143.253:45437: connect: Connection refused (error 111)
22:11:54.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:54.213104 8050 leader_election.cc:302] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 8e8c40d6791a43a99f67354e2df309ee; no voters: 65fbe4442b8b4d21af2fa7d6b162abb7, 929b15eec5ef4bd9adb23cb19aea80f2
22:11:54.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:54.213779 8284 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
22:11:54.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:54.314366 8117 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:11:54.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:54.314684 8117 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:11:55.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 22:11:55 dist-test-slave-qzth krb5kdc[7755](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790547115, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.143.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
22:11:55.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:55.008772 8045 init.cc:295] Successfully reacquired a new kerberos TGT
22:11:55.037 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.143.254:40351(127.7.143.254:40351)] connecting to peer
22:11:55.039 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.143.253:45437(127.7.143.253:45437)] connecting to peer
22:11:55.039 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:11:55.040 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.143.254:40351(127.7.143.254:40351)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:11:55.040 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.143.254:40351: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:11:55.042 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:11:55.042 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.143.253:45437(127.7.143.253:45437)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:11:55.042 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.143.253:45437: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:11:55.043 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.143.252:41589,127.7.143.254:40351,127.7.143.253:45437; will retry
22:11:55.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:55.300750 8030 master_runner.cc:186] Error getting master registration for 127.7.143.254:40351: Network error: Client connection negotiation failed: client connection to 127.7.143.254:40351: connect: Connection refused (error 111),
22:11:55.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:55.301100 8030 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
22:11:55.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:55.317466 8117 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:11:55.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:55.317727 8117 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:11:55.357 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.143.254:40351(127.7.143.254:40351)] connecting to peer
22:11:55.359 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:11:55.359 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.143.254:40351(127.7.143.254:40351)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:11:55.359 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.143.253:45437(127.7.143.253:45437)] connecting to peer
22:11:55.360 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.143.254:40351: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:11:55.361 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:11:55.361 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.143.253:45437(127.7.143.253:45437)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:11:55.362 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.143.253:45437: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:11:55.362 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.143.252:41589,127.7.143.254:40351,127.7.143.253:45437; will retry
22:11:56.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:56.320351 8117 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:11:56.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:56.320597 8117 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:11:56.417 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.143.254:40351(127.7.143.254:40351)] connecting to peer
22:11:56.419 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.143.253:45437(127.7.143.253:45437)] connecting to peer
22:11:56.419 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:11:56.419 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.143.254:40351(127.7.143.254:40351)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:11:56.420 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.143.254:40351: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:11:56.421 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:11:56.421 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.143.253:45437(127.7.143.253:45437)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:11:56.422 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.143.253:45437: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:11:56.422 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.143.252:41589,127.7.143.254:40351,127.7.143.253:45437; will retry
22:11:57.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:57.323267 8117 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:11:57.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:57.323490 8117 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:11:57.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:57.474839 8292 raft_consensus.cc:482] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
22:11:57.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:57.475188 8292 raft_consensus.cc:504] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "929b15eec5ef4bd9adb23cb19aea80f2" member_type: VOTER last_known_addr { host: "127.7.143.254" port: 40351 } } peers { permanent_uuid: "65fbe4442b8b4d21af2fa7d6b162abb7" member_type: VOTER last_known_addr { host: "127.7.143.253" port: 45437 } } peers { permanent_uuid: "8e8c40d6791a43a99f67354e2df309ee" member_type: VOTER last_known_addr { host: "127.7.143.252" port: 41589 } }
22:11:57.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:57.476573 8292 leader_election.cc:288] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 929b15eec5ef4bd9adb23cb19aea80f2 (127.7.143.254:40351), 65fbe4442b8b4d21af2fa7d6b162abb7 (127.7.143.253:45437)
22:11:57.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:57.481400 8050 leader_election.cc:334] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 929b15eec5ef4bd9adb23cb19aea80f2 (127.7.143.254:40351): Network error: Client connection negotiation failed: client connection to 127.7.143.254:40351: connect: Connection refused (error 111)
22:11:57.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:57.481977 8050 leader_election.cc:334] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 65fbe4442b8b4d21af2fa7d6b162abb7 (127.7.143.253:45437): Network error: Client connection negotiation failed: client connection to 127.7.143.253:45437: connect: Connection refused (error 111)
22:11:57.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:57.482203 8050 leader_election.cc:302] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 8e8c40d6791a43a99f67354e2df309ee; no voters: 65fbe4442b8b4d21af2fa7d6b162abb7, 929b15eec5ef4bd9adb23cb19aea80f2
22:11:57.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:57.482730 8292 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
22:11:58.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:58.326113 8117 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:11:58.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:58.326339 8117 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:11:59.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:59.328819 8117 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:11:59.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:59.329035 8117 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:11:59.677 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.143.254:40351(127.7.143.254:40351)] connecting to peer
22:11:59.678 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.143.253:45437(127.7.143.253:45437)] connecting to peer
22:11:59.679 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:11:59.679 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.143.254:40351(127.7.143.254:40351)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:11:59.680 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.143.254:40351: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:11:59.682 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:11:59.682 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.143.253:45437(127.7.143.253:45437)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:11:59.682 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.143.253:45437: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:11:59.682 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.143.252:41589,127.7.143.254:40351,127.7.143.253:45437; will retry
22:11:59.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:59.934031 8297 raft_consensus.cc:482] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
22:11:59.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:59.934440 8297 raft_consensus.cc:504] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "929b15eec5ef4bd9adb23cb19aea80f2" member_type: VOTER last_known_addr { host: "127.7.143.254" port: 40351 } } peers { permanent_uuid: "65fbe4442b8b4d21af2fa7d6b162abb7" member_type: VOTER last_known_addr { host: "127.7.143.253" port: 45437 } } peers { permanent_uuid: "8e8c40d6791a43a99f67354e2df309ee" member_type: VOTER last_known_addr { host: "127.7.143.252" port: 41589 } }
22:11:59.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:59.936281 8297 leader_election.cc:288] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 929b15eec5ef4bd9adb23cb19aea80f2 (127.7.143.254:40351), 65fbe4442b8b4d21af2fa7d6b162abb7 (127.7.143.253:45437)
22:11:59.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:59.938786 8050 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.143.254:40351: connect: Connection refused (error 111) [suppressed 3 similar messages]
22:11:59.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:59.941794 8050 leader_election.cc:334] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 929b15eec5ef4bd9adb23cb19aea80f2 (127.7.143.254:40351): Network error: Client connection negotiation failed: client connection to 127.7.143.254:40351: connect: Connection refused (error 111)
22:11:59.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:11:59.942502 8050 leader_election.cc:334] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 65fbe4442b8b4d21af2fa7d6b162abb7 (127.7.143.253:45437): Network error: Client connection negotiation failed: client connection to 127.7.143.253:45437: connect: Connection refused (error 111)
22:11:59.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:59.942778 8050 leader_election.cc:302] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 8e8c40d6791a43a99f67354e2df309ee; no voters: 65fbe4442b8b4d21af2fa7d6b162abb7, 929b15eec5ef4bd9adb23cb19aea80f2
22:11:59.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:11:59.943459 8297 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
22:12:00.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:12:00.302917 8030 master_runner.cc:186] Error getting master registration for 127.7.143.254:40351: Network error: Client connection negotiation failed: client connection to 127.7.143.254:40351: connect: Connection refused (error 111),
22:12:00.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:12:00.303261 8030 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
22:12:00.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:12:00.332180 8117 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:12:00.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:12:00.332392 8117 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:12:01.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:12:01.335225 8117 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:12:01.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:12:01.335461 8117 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:12:01.918 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.143.254:40351(127.7.143.254:40351)] connecting to peer
22:12:01.919 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.143.253:45437(127.7.143.253:45437)] connecting to peer
22:12:01.920 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:12:01.920 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.143.254:40351(127.7.143.254:40351)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:12:01.921 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.143.254:40351: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:12:01.922 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:12:01.922 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.143.253:45437(127.7.143.253:45437)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:12:01.922 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.143.253:45437: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:12:01.922 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.143.252:41589,127.7.143.254:40351,127.7.143.253:45437; will retry
22:12:02.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:12:02.338227 8117 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:12:02.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:12:02.338452 8117 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:12:03.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:12:03.340963 8117 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:12:03.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:12:03.341188 8117 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:12:04.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:12:04.345165 8117 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:12:04.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:12:04.345563 8117 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:12:04.958 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.143.254:40351(127.7.143.254:40351)] connecting to peer
22:12:04.959 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.143.253:45437(127.7.143.253:45437)] connecting to peer
22:12:04.960 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:12:04.960 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.143.254:40351(127.7.143.254:40351)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:12:04.961 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.143.254:40351: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:12:04.961 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:12:04.961 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.143.253:45437(127.7.143.253:45437)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:12:04.961 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.143.253:45437: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:12:04.962 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.143.252:41589,127.7.143.254:40351,127.7.143.253:45437; will retry
22:12:05.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 22:12:05 dist-test-slave-qzth krb5kdc[7755](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790547125, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.143.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
22:12:05.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:12:05.011087 8045 init.cc:295] Successfully reacquired a new kerberos TGT
22:12:05.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:12:05.306334 8030 master_runner.cc:186] Error getting master registration for 127.7.143.254:40351: Network error: Client connection negotiation failed: client connection to 127.7.143.254:40351: connect: Connection refused (error 111),
22:12:05.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:12:05.306689 8030 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
22:12:05.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:12:05.349613 8117 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:12:05.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:12:05.349933 8117 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:12:05.418 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.143.254:40351(127.7.143.254:40351)] connecting to peer
22:12:05.418 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.143.253:45437(127.7.143.253:45437)] connecting to peer
22:12:05.419 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:12:05.419 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.143.254:40351(127.7.143.254:40351)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:12:05.419 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.143.254:40351: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:12:05.419 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:12:05.419 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.143.253:45437(127.7.143.253:45437)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:12:05.419 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.143.253:45437: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:12:05.421 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.143.252:41589,127.7.143.254:40351,127.7.143.253:45437; will retry
22:12:06.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:12:06.352499 8117 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:12:06.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:12:06.352766 8117 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:12:06.838 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.143.254:40351(127.7.143.254:40351)] connecting to peer
22:12:06.838 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.143.253:45437(127.7.143.253:45437)] connecting to peer
22:12:06.839 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:12:06.839 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.143.254:40351(127.7.143.254:40351)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:12:06.839 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.143.254:40351: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:12:06.839 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:12:06.839 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.143.253:45437(127.7.143.253:45437)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:12:06.839 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.143.253:45437: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:12:06.841 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.143.252:41589,127.7.143.254:40351,127.7.143.253:45437; will retry
22:12:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:12:07.355602 8117 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:12:07.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:12:07.355906 8117 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:12:08.358 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.143.254:40351(127.7.143.254:40351)] connecting to peer
22:12:08.359 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.143.253:45437(127.7.143.253:45437)] connecting to peer
22:12:08.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:12:08.359382 8117 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:12:08.359 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:12:08.359 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.143.254:40351(127.7.143.254:40351)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:12:08.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:12:08.359853 8117 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:12:08.360 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.143.254:40351: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:12:08.360 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:12:08.361 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.143.253:45437(127.7.143.253:45437)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:12:08.361 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.143.253:45437: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:12:08.362 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.143.252:41589,127.7.143.254:40351,127.7.143.253:45437; will retry
22:12:08.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:12:08.886765 8312 raft_consensus.cc:482] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
22:12:08.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:12:08.887178 8312 raft_consensus.cc:504] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "929b15eec5ef4bd9adb23cb19aea80f2" member_type: VOTER last_known_addr { host: "127.7.143.254" port: 40351 } } peers { permanent_uuid: "65fbe4442b8b4d21af2fa7d6b162abb7" member_type: VOTER last_known_addr { host: "127.7.143.253" port: 45437 } } peers { permanent_uuid: "8e8c40d6791a43a99f67354e2df309ee" member_type: VOTER last_known_addr { host: "127.7.143.252" port: 41589 } }
22:12:08.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:12:08.889072 8312 leader_election.cc:288] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 929b15eec5ef4bd9adb23cb19aea80f2 (127.7.143.254:40351), 65fbe4442b8b4d21af2fa7d6b162abb7 (127.7.143.253:45437)
22:12:08.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:12:08.891191 8050 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.143.254:40351: connect: Connection refused (error 111) [suppressed 1 similar messages]
22:12:08.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:12:08.894053 8050 leader_election.cc:334] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 929b15eec5ef4bd9adb23cb19aea80f2 (127.7.143.254:40351): Network error: Client connection negotiation failed: client connection to 127.7.143.254:40351: connect: Connection refused (error 111)
22:12:08.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:12:08.894620 8050 leader_election.cc:334] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 65fbe4442b8b4d21af2fa7d6b162abb7 (127.7.143.253:45437): Network error: Client connection negotiation failed: client connection to 127.7.143.253:45437: connect: Connection refused (error 111)
22:12:08.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:12:08.894858 8050 leader_election.cc:302] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 8e8c40d6791a43a99f67354e2df309ee; no voters: 65fbe4442b8b4d21af2fa7d6b162abb7, 929b15eec5ef4bd9adb23cb19aea80f2
22:12:08.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:12:08.895324 8312 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
22:12:09.238 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.143.254:40351(127.7.143.254:40351)] connecting to peer
22:12:09.238 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.143.253:45437(127.7.143.253:45437)] connecting to peer
22:12:09.238 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:12:09.239 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.143.254:40351(127.7.143.254:40351)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:12:09.239 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.143.254:40351: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:12:09.239 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:12:09.239 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.143.253:45437(127.7.143.253:45437)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:12:09.240 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.143.253:45437: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:12:09.241 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.143.252:41589,127.7.143.254:40351,127.7.143.253:45437; will retry
22:12:09.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:12:09.362821 8117 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:12:09.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:12:09.363082 8117 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:12:09.538 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.143.254:40351(127.7.143.254:40351)] connecting to peer
22:12:09.539 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.143.253:45437(127.7.143.253:45437)] connecting to peer
22:12:09.539 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:12:09.540 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.143.254:40351(127.7.143.254:40351)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:12:09.540 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:12:09.540 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.143.254:40351: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:12:09.540 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.143.253:45437(127.7.143.253:45437)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:12:09.540 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.143.253:45437: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:12:09.541 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.143.252:41589,127.7.143.254:40351,127.7.143.253:45437; will retry
22:12:09.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:12:09.632441 7248 debug-util.cc:398] Leaking SignalData structure 0x7b08000b9b00 after lost signal to thread 7180
22:12:09.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:12:09.633098 7248 debug-util.cc:398] Leaking SignalData structure 0x7b08000a1800 after lost signal to thread 7251
22:12:10.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:12:10.309077 8030 master_runner.cc:186] Error getting master registration for 127.7.143.254:40351: Network error: Client connection negotiation failed: client connection to 127.7.143.254:40351: connect: Connection refused (error 111),
22:12:10.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:12:10.309387 8030 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
22:12:10.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:12:10.366045 8117 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:12:10.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:12:10.366369 8117 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:12:11.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:12:11.369014 8117 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:12:11.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:12:11.369258 8117 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:12:12.018 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.143.254:40351(127.7.143.254:40351)] connecting to peer
22:12:12.020 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:12:12.020 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.143.254:40351(127.7.143.254:40351)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:12:12.020 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.143.253:45437(127.7.143.253:45437)] connecting to peer
22:12:12.020 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.143.254:40351: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:12:12.022 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:12:12.022 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.143.253:45437(127.7.143.253:45437)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:12:12.022 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.143.253:45437: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:12:12.022 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.143.252:41589,127.7.143.254:40351,127.7.143.253:45437; will retry
22:12:12.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:12:12.371968 8117 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:12:12.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:12:12.372184 8117 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:12:13.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:12:13.374835 8117 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:12:13.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:12:13.375075 8117 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:12:14.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:12:14.378518 8117 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:12:14.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:12:14.378834 8117 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:12:14.917 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.143.254:40351(127.7.143.254:40351)] connecting to peer
22:12:14.918 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.143.253:45437(127.7.143.253:45437)] connecting to peer
22:12:14.918 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:12:14.918 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.143.254:40351(127.7.143.254:40351)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:12:14.918 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.143.254:40351: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:12:14.919 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:12:14.919 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.143.253:45437(127.7.143.253:45437)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:12:14.919 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.143.253:45437: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:12:14.921 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.143.252:41589,127.7.143.254:40351,127.7.143.253:45437; will retry
22:12:15.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 22:12:15 dist-test-slave-qzth krb5kdc[7755](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790547135, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.143.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
22:12:15.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:12:15.013507 8045 init.cc:295] Successfully reacquired a new kerberos TGT
22:12:15.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:12:15.312878 8030 master_runner.cc:186] Error getting master registration for 127.7.143.254:40351: Network error: Client connection negotiation failed: client connection to 127.7.143.254:40351: connect: Connection refused (error 111),
22:12:15.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:12:15.313265 8030 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
22:12:15.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:12:15.383659 8117 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:12:15.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:12:15.383968 8117 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:12:16.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:12:16.374955 7094 debug-util.cc:398] Leaking SignalData structure 0x7b080009dd20 after lost signal to thread 7031
22:12:16.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:12:16.375617 7094 debug-util.cc:398] Leaking SignalData structure 0x7b080009dd00 after lost signal to thread 7097
22:12:16.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:12:16.387693 8117 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:12:16.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:12:16.387980 8117 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:12:16.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:12:16.951674 7445 debug-util.cc:398] Leaking SignalData structure 0x7b0800041140 after lost signal to thread 7311
22:12:16.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:12:16.952404 7445 debug-util.cc:398] Leaking SignalData structure 0x7b08000b3aa0 after lost signal to thread 7448
22:12:17.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:12:17.391584 8117 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:12:17.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:12:17.391911 8117 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:12:18.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:12:18.394259 8117 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8e8c40d6791a43a99f67354e2df309ee: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:12:18.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:12:18.394479 8117 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:12:18.658 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.143.254:40351(127.7.143.254:40351)] connecting to peer
22:12:18.658 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.143.253:45437(127.7.143.253:45437)] connecting to peer
22:12:18.659 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:12:18.659 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.143.254:40351(127.7.143.254:40351)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:12:18.659 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.143.254:40351: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351
22:12:18.659 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:12:18.659 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.143.253:45437(127.7.143.253:45437)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:12:18.660 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.143.253:45437: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
22:12:18.661 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.143.252:41589,127.7.143.254:40351,127.7.143.253:45437; will retry
22:12:18.666 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=28, TimeoutTracker(timeout=30000, elapsed=29746), Traces: [0ms] refreshing cache from master, [35ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.252:41589, [106ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.254:40351, [108ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.253:45437, [168ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.254:40351: Network error: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351, [217ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.253:45437: Network error: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [481ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.252:41589: OK, [484ms] delaying RPC due to: Service unavailable: Master config (127.7.143.252:41589,127.7.143.254:40351,127.7.143.253:45437) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [497ms] refreshing cache from master, [498ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.252:41589, [499ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.254:40351, [502ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.253:45437, [503ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.254:40351: Network error: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351, [503ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.252:41589: OK, [505ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.253:45437: Network error: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [506ms] delaying RPC due to: Service unavailable: Master config (127.7.143.252:41589,127.7.143.254:40351,127.7.143.253:45437) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [517ms] refreshing cache from master, [517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.252:41589, [517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.254:40351, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.253:45437, [520ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.254:40351: Network error: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351, [522ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.252:41589: OK, [523ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.253:45437: Network error: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [524ms] delaying RPC due to: Service unavailable: Master config (127.7.143.252:41589,127.7.143.254:40351,127.7.143.253:45437) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [536ms] refreshing cache from master, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.252:41589, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.254:40351, [538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.253:45437, [541ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.254:40351: Network error: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351, [541ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.252:41589: OK, [542ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.253:45437: Network error: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [543ms] delaying RPC due to: Service unavailable: Master config (127.7.143.252:41589,127.7.143.254:40351,127.7.143.253:45437) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [557ms] refreshing cache from master, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.252:41589, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.254:40351, [558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.253:45437, [560ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.254:40351: Network error: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351, [561ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.252:41589: OK, [562ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.253:45437: Network error: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [563ms] delaying RPC due to: Service unavailable: Master config (127.7.143.252:41589,127.7.143.254:40351,127.7.143.253:45437) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [577ms] refreshing cache from master, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.252:41589, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.254:40351, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.253:45437, [581ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.252:41589: OK, [581ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.254:40351: Network error: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351, [582ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.253:45437: Network error: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [583ms] delaying RPC due to: Service unavailable: Master config (127.7.143.252:41589,127.7.143.254:40351,127.7.143.253:45437) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [596ms] refreshing cache from master, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.252:41589, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.254:40351, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.253:45437, [600ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.254:40351: Network error: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351, [601ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.252:41589: OK, [602ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.253:45437: Network error: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [603ms] delaying RPC due to: Service unavailable: Master config (127.7.143.252:41589,127.7.143.254:40351,127.7.143.253:45437) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [717ms] refreshing cache from master, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.252:41589, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.254:40351, [719ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.253:45437, [720ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.254:40351: Network error: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351, [722ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.252:41589: OK, [723ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.253:45437: Network error: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [724ms] delaying RPC due to: Service unavailable: Master config (127.7.143.252:41589,127.7.143.254:40351,127.7.143.253:45437) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [737ms] refreshing cache from master, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.252:41589, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.254:40351, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.253:45437, [741ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.254:40351: Network error: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351, [741ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.252:41589: OK, [742ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.253:45437: Network error: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [743ms] delaying RPC due to: Service unavailable: Master config (127.7.143.252:41589,127.7.143.254:40351,127.7.143.253:45437) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [897ms] refreshing cache from master, [897ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.252:41589, [898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.254:40351, [898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.253:45437, [901ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.254:40351: Network error: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351, [901ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.252:41589: OK, [902ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.253:45437: Network error: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [904ms] delaying RPC due to: Service unavailable: Master config (127.7.143.252:41589,127.7.143.254:40351,127.7.143.253:45437) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [1116ms] refreshing cache from master, [1117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.252:41589, [1117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.254:40351, [1118ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.253:45437, [1121ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.254:40351: Network error: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351, [1122ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.252:41589: OK, [1122ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.253:45437: Network error: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [1123ms] delaying RPC due to: Service unavailable: Master config (127.7.143.252:41589,127.7.143.254:40351,127.7.143.253:45437) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [1916ms] refreshing cache from master, [1916ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.252:41589, [1917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.254:40351, [1918ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.253:45437, [1920ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.254:40351: Network error: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351, [1920ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.253:45437: Network error: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [1921ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.252:41589: OK, [1921ms] delaying RPC due to: Service unavailable: Master config (127.7.143.252:41589,127.7.143.254:40351,127.7.143.253:45437) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [2057ms] refreshing cache from master, [2057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.252:41589, [2058ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.254:40351, [2059ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.253:45437, [2061ms] too many traces: truncated at 100 traces, deferred=Deferred@1619225268(state=PENDING, result=null, callback=(continuation of Deferred@1546163120 after retry RPC after error@360121393) -> (continuation of Deferred@1164840193 after retry RPC after error@1044584914) -> (continuation of Deferred@644786361 after retry RPC after error@185956041) -> (continuation of Deferred@1042707631 after retry RPC after error@403535664) -> (continuation of Deferred@1265971176 after retry RPC after error@1802939249) -> (continuation of Deferred@1727539644 after retry RPC after error@364865188) -> (continuation of Deferred@2010583444 after retry RPC after error@690225468) -> (continuation of Deferred@238450797 after retry RPC after error@1624417609) -> (continuation of Deferred@801526646 after retry RPC after error@652380755) -> (continuation of Deferred@455769074 after retry RPC after error@1856470323) -> (continuation of Deferred@1936186994 after retry RPC after error@1097167455) -> (continuation of Deferred@156791914 after retry RPC after error@1556340375) -> (continuation of Deferred@286924595 after retry RPC after error@2064085189) -> (continuation of Deferred@825663723 after retry RPC after error@1942218556) -> (continuation of Deferred@845254128 after retry RPC after error@1923109409) -> (continuation of Deferred@1989514598 after retry RPC after error@436430479) -> (continuation of Deferred@607348274 after retry RPC after error@1416759759) -> (continuation of Deferred@244750466 after retry RPC after error@262185153) -> (continuation of Deferred@807224817 after retry RPC after error@1961168460) -> (continuation of Deferred@1317915354 after retry RPC after error@1151458658) -> (continuation of Deferred@508111562 after retry RPC after error@1107476513) -> (continuation of Deferred@1652099489 after retry RPC after error@679055937) -> (continuation of Deferred@2142364414 after retry RPC after error@736503521) -> (continuation of Deferred@33035837 after retry RPC after error@1702324185) -> (continuation of Deferred@354541467 after retry RPC after error@267006704) -> (continuation of Deferred@1315083470 after retry RPC after error@284523380) -> (continuation of Deferred@196326390 after retry RPC after error@1427144130), errback=(continuation of Deferred@1546163120 after retry RPC after error@360121393) -> (continuation of Deferred@1164840193 after retry RPC after error@1044584914) -> (continuation of Deferred@644786361 after retry RPC after error@185956041) -> (continuation of Deferred@1042707631 after retry RPC after error@403535664) -> (continuation of Deferred@1265971176 after retry RPC after error@1802939249) -> (continuation of Deferred@1727539644 after retry RPC after error@364865188) -> (continuation of Deferred@2010583444 after retry RPC after error@690225468) -> (continuation of Deferred@238450797 after retry RPC after error@1624417609) -> (continuation of Deferred@801526646 after retry RPC after error@652380755) -> (continuation of Deferred@455769074 after retry RPC after error@1856470323) -> (continuation of Deferred@1936186994 after retry RPC after error@1097167455) -> (continuation of Deferred@156791914 after retry RPC after error@1556340375) -> (continuation of Deferred@286924595 after retry RPC after error@2064085189) -> (continuation of Deferred@825663723 after retry RPC after error@1942218556) -> (continuation of Deferred@845254128 after retry RPC after error@1923109409) -> (continuation of Deferred@1989514598 after retry RPC after error@436430479) -> (continuation of Deferred@607348274 after retry RPC after error@1416759759) -> (continuation of Deferred@244750466 after retry RPC after error@262185153) -> (continuation of Deferred@807224817 after retry RPC after error@1961168460) -> (continuation of Deferred@1317915354 after retry RPC after error@1151458658) -> (continuation of Deferred@508111562 after retry RPC after error@1107476513) -> (continuation of Deferred@1652099489 after retry RPC after error@679055937) -> (continuation of Deferred@2142364414 after retry RPC after error@736503521) -> (continuation of Deferred@33035837 after retry RPC after error@1702324185) -> (continuation of Deferred@354541467 after retry RPC after error@267006704) -> (continuation of Deferred@1315083470 after retry RPC after error@284523380) -> (continuation of Deferred@196326390 after retry RPC after error@1427144130)))
22:12:18.671 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.7.143.253:45437(127.7.143.253:45437)] disconnecting while in state TERMINATED
22:12:18.672 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.7.143.254:40351(127.7.143.254:40351)] disconnecting while in state TERMINATED
22:12:18.672 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.7.143.252:41589(127.7.143.252:41589)] disconnecting while in state READY
22:12:18.676 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
22:12:18.678 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
22:12:18.678 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.7.143.252:41589(127.7.143.252:41589)] handling channelInactive
22:12:18.679 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.143.252:41589(127.7.143.252:41589)] cleaning up while in state READY due to: connection closed
22:12:18.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:12:18.683311 7743 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu with pid 8030
22:12:19.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T22:12:19Z chronyd exiting
22:12:20.885 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
22:12:20.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:12:20.886534 7018 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu with pid 7297
22:12:20.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:12:20.912009 7018 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu with pid 7450
22:12:20.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:12:20.937247 7018 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu with pid 7600
22:12:20.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:12:20.970486 7018 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu with pid 7030
22:12:21.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:12:21.000161 7018 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu with pid 7100
22:12:21.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:12:21.028656 7018 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu with pid 7175
22:12:22.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T22:12:22Z chronyd exiting
22:12:23.202 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=28, TimeoutTracker(timeout=30000, elapsed=29746), Traces: [0ms] refreshing cache from master, [35ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.252:41589, [106ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.254:40351, [108ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.253:45437, [168ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.254:40351: Network error: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351, [217ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.253:45437: Network error: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [481ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.252:41589: OK, [484ms] delaying RPC due to: Service unavailable: Master config (127.7.143.252:41589,127.7.143.254:40351,127.7.143.253:45437) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [497ms] refreshing cache from master, [498ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.252:41589, [499ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.254:40351, [502ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.253:45437, [503ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.254:40351: Network error: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351, [503ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.252:41589: OK, [505ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.253:45437: Network error: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [506ms] delaying RPC due to: Service unavailable: Master config (127.7.143.252:41589,127.7.143.254:40351,127.7.143.253:45437) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [517ms] refreshing cache from master, [517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.252:41589, [517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.254:40351, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.253:45437, [520ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.254:40351: Network error: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351, [522ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.252:41589: OK, [523ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.253:45437: Network error: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [524ms] delaying RPC due to: Service unavailable: Master config (127.7.143.252:41589,127.7.143.254:40351,127.7.143.253:45437) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [536ms] refreshing cache from master, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.252:41589, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.254:40351, [538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.253:45437, [541ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.254:40351: Network error: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351, [541ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.252:41589: OK, [542ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.253:45437: Network error: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [543ms] delaying RPC due to: Service unavailable: Master config (127.7.143.252:41589,127.7.143.254:40351,127.7.143.253:45437) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [557ms] refreshing cache from master, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.252:41589, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.254:40351, [558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.253:45437, [560ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.254:40351: Network error: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351, [561ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.252:41589: OK, [562ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.253:45437: Network error: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [563ms] delaying RPC due to: Service unavailable: Master config (127.7.143.252:41589,127.7.143.254:40351,127.7.143.253:45437) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [577ms] refreshing cache from master, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.252:41589, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.254:40351, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.253:45437, [581ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.252:41589: OK, [581ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.254:40351: Network error: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351, [582ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.253:45437: Network error: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [583ms] delaying RPC due to: Service unavailable: Master config (127.7.143.252:41589,127.7.143.254:40351,127.7.143.253:45437) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [596ms] refreshing cache from master, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.252:41589, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.254:40351, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.253:45437, [600ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.254:40351: Network error: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351, [601ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.252:41589: OK, [602ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.253:45437: Network error: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [603ms] delaying RPC due to: Service unavailable: Master config (127.7.143.252:41589,127.7.143.254:40351,127.7.143.253:45437) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [717ms] refreshing cache from master, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.252:41589, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.254:40351, [719ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.253:45437, [720ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.254:40351: Network error: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351, [722ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.252:41589: OK, [723ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.253:45437: Network error: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [724ms] delaying RPC due to: Service unavailable: Master config (127.7.143.252:41589,127.7.143.254:40351,127.7.143.253:45437) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [737ms] refreshing cache from master, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.252:41589, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.254:40351, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.253:45437, [741ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.254:40351: Network error: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351, [741ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.252:41589: OK, [742ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.253:45437: Network error: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [743ms] delaying RPC due to: Service unavailable: Master config (127.7.143.252:41589,127.7.143.254:40351,127.7.143.253:45437) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [897ms] refreshing cache from master, [897ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.252:41589, [898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.254:40351, [898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.253:45437, [901ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.254:40351: Network error: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351, [901ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.252:41589: OK, [902ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.253:45437: Network error: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [904ms] delaying RPC due to: Service unavailable: Master config (127.7.143.252:41589,127.7.143.254:40351,127.7.143.253:45437) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [1116ms] refreshing cache from master, [1117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.252:41589, [1117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.254:40351, [1118ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.253:45437, [1121ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.254:40351: Network error: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351, [1122ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.252:41589: OK, [1122ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.253:45437: Network error: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [1123ms] delaying RPC due to: Service unavailable: Master config (127.7.143.252:41589,127.7.143.254:40351,127.7.143.253:45437) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [1916ms] refreshing cache from master, [1916ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.252:41589, [1917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.254:40351, [1918ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.253:45437, [1920ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.254:40351: Network error: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351, [1920ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.253:45437: Network error: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [1921ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.252:41589: OK, [1921ms] delaying RPC due to: Service unavailable: Master config (127.7.143.252:41589,127.7.143.254:40351,127.7.143.253:45437) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [2057ms] refreshing cache from master, [2057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.252:41589, [2058ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.254:40351, [2059ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.253:45437, [2061ms] too many traces: truncated at 100 traces, deferred=Deferred@1619225268(state=PENDING, result=null, callback=(continuation of Deferred@1546163120 after retry RPC after error@360121393) -> (continuation of Deferred@1164840193 after retry RPC after error@1044584914) -> (continuation of Deferred@644786361 after retry RPC after error@185956041) -> (continuation of Deferred@1042707631 after retry RPC after error@403535664) -> (continuation of Deferred@1265971176 after retry RPC after error@1802939249) -> (continuation of Deferred@1727539644 after retry RPC after error@364865188) -> (continuation of Deferred@2010583444 after retry RPC after error@690225468) -> (continuation of Deferred@238450797 after retry RPC after error@1624417609) -> (continuation of Deferred@801526646 after retry RPC after error@652380755) -> (continuation of Deferred@455769074 after retry RPC after error@1856470323) -> (continuation of Deferred@1936186994 after retry RPC after error@1097167455) -> (continuation of Deferred@156791914 after retry RPC after error@1556340375) -> (continuation of Deferred@286924595 after retry RPC after error@2064085189) -> (continuation of Deferred@825663723 after retry RPC after error@1942218556) -> (continuation of Deferred@845254128 after retry RPC after error@1923109409) -> (continuation of Deferred@1989514598 after retry RPC after error@436430479) -> (continuation of Deferred@607348274 after retry RPC after error@1416759759) -> (continuation of Deferred@244750466 after retry RPC after error@262185153) -> (continuation of Deferred@807224817 after retry RPC after error@1961168460) -> (continuation of Deferred@1317915354 after retry RPC after error@1151458658) -> (continuation of Deferred@508111562 after retry RPC after error@1107476513) -> (continuation of Deferred@1652099489 after retry RPC after error@679055937) -> (continuation of Deferred@2142364414 after retry RPC after error@736503521) -> (continuation of Deferred@33035837 after retry RPC after error@1702324185) -> (continuation of Deferred@354541467 after retry RPC after error@267006704) -> (continuation of Deferred@1315083470 after retry RPC after error@284523380) -> (continuation of Deferred@196326390 after retry RPC after error@1427144130), errback=(continuation of Deferred@1546163120 after retry RPC after error@360121393) -> (continuation of Deferred@1164840193 after retry RPC after error@1044584914) -> (continuation of Deferred@644786361 after retry RPC after error@185956041) -> (continuation of Deferred@1042707631 after retry RPC after error@403535664) -> (continuation of Deferred@1265971176 after retry RPC after error@1802939249) -> (continuation of Deferred@1727539644 after retry RPC after error@364865188) -> (continuation of Deferred@2010583444 after retry RPC after error@690225468) -> (continuation of Deferred@238450797 after retry RPC after error@1624417609) -> (continuation of Deferred@801526646 after retry RPC after error@652380755) -> (continuation of Deferred@455769074 after retry RPC after error@1856470323) -> (continuation of Deferred@1936186994 after retry RPC after error@1097167455) -> (continuation of Deferred@156791914 after retry RPC after error@1556340375) -> (continuation of Deferred@286924595 after retry RPC after error@2064085189) -> (continuation of Deferred@825663723 after retry RPC after error@1942218556) -> (continuation of Deferred@845254128 after retry RPC after error@1923109409) -> (continuation of Deferred@1989514598 after retry RPC after error@436430479) -> (continuation of Deferred@607348274 after retry RPC after error@1416759759) -> (continuation of Deferred@244750466 after retry RPC after error@262185153) -> (continuation of Deferred@807224817 after retry RPC after error@1961168460) -> (continuation of Deferred@1317915354 after retry RPC after error@1151458658) -> (continuation of Deferred@508111562 after retry RPC after error@1107476513) -> (continuation of Deferred@1652099489 after retry RPC after error@679055937) -> (continuation of Deferred@2142364414 after retry RPC after error@736503521) -> (continuation of Deferred@33035837 after retry RPC after error@1702324185) -> (continuation of Deferred@354541467 after retry RPC after error@267006704) -> (continuation of Deferred@1315083470 after retry RPC after error@284523380) -> (continuation of Deferred@196326390 after retry RPC after error@1427144130)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.7.143.252:41589,127.7.143.254:40351,127.7.143.253:45437) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.7.143.253:45437
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=28, TimeoutTracker(timeout=30000, elapsed=29746), Traces: [0ms] refreshing cache from master, [35ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.252:41589, [106ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.254:40351, [108ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.253:45437, [168ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.254:40351: Network error: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351, [217ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.253:45437: Network error: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [481ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.252:41589: OK, [484ms] delaying RPC due to: Service unavailable: Master config (127.7.143.252:41589,127.7.143.254:40351,127.7.143.253:45437) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [497ms] refreshing cache from master, [498ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.252:41589, [499ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.254:40351, [502ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.253:45437, [503ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.254:40351: Network error: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351, [503ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.252:41589: OK, [505ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.253:45437: Network error: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [506ms] delaying RPC due to: Service unavailable: Master config (127.7.143.252:41589,127.7.143.254:40351,127.7.143.253:45437) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [517ms] refreshing cache from master, [517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.252:41589, [517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.254:40351, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.253:45437, [520ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.254:40351: Network error: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351, [522ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.252:41589: OK, [523ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.253:45437: Network error: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [524ms] delaying RPC due to: Service unavailable: Master config (127.7.143.252:41589,127.7.143.254:40351,127.7.143.253:45437) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [536ms] refreshing cache from master, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.252:41589, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.254:40351, [538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.253:45437, [541ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.254:40351: Network error: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351, [541ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.252:41589: OK, [542ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.253:45437: Network error: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [543ms] delaying RPC due to: Service unavailable: Master config (127.7.143.252:41589,127.7.143.254:40351,127.7.143.253:45437) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [557ms] refreshing cache from master, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.252:41589, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.254:40351, [558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.253:45437, [560ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.254:40351: Network error: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351, [561ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.252:41589: OK, [562ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.253:45437: Network error: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [563ms] delaying RPC due to: Service unavailable: Master config (127.7.143.252:41589,127.7.143.254:40351,127.7.143.253:45437) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [577ms] refreshing cache from master, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.252:41589, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.254:40351, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.253:45437, [581ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.252:41589: OK, [581ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.254:40351: Network error: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351, [582ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.253:45437: Network error: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [583ms] delaying RPC due to: Service unavailable: Master config (127.7.143.252:41589,127.7.143.254:40351,127.7.143.253:45437) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [596ms] refreshing cache from master, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.252:41589, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.254:40351, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.253:45437, [600ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.254:40351: Network error: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351, [601ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.252:41589: OK, [602ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.253:45437: Network error: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [603ms] delaying RPC due to: Service unavailable: Master config (127.7.143.252:41589,127.7.143.254:40351,127.7.143.253:45437) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [717ms] refreshing cache from master, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.252:41589, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.254:40351, [719ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.253:45437, [720ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.254:40351: Network error: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351, [722ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.252:41589: OK, [723ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.253:45437: Network error: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [724ms] delaying RPC due to: Service unavailable: Master config (127.7.143.252:41589,127.7.143.254:40351,127.7.143.253:45437) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [737ms] refreshing cache from master, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.252:41589, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.254:40351, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.253:45437, [741ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.254:40351: Network error: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351, [741ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.252:41589: OK, [742ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.253:45437: Network error: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [743ms] delaying RPC due to: Service unavailable: Master config (127.7.143.252:41589,127.7.143.254:40351,127.7.143.253:45437) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [897ms] refreshing cache from master, [897ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.252:41589, [898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.254:40351, [898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.253:45437, [901ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.254:40351: Network error: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351, [901ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.252:41589: OK, [902ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.253:45437: Network error: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [904ms] delaying RPC due to: Service unavailable: Master config (127.7.143.252:41589,127.7.143.254:40351,127.7.143.253:45437) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [1116ms] refreshing cache from master, [1117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.252:41589, [1117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.254:40351, [1118ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.253:45437, [1121ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.254:40351: Network error: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351, [1122ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.252:41589: OK, [1122ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.253:45437: Network error: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [1123ms] delaying RPC due to: Service unavailable: Master config (127.7.143.252:41589,127.7.143.254:40351,127.7.143.253:45437) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [1916ms] refreshing cache from master, [1916ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.252:41589, [1917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.254:40351, [1918ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.253:45437, [1920ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.254:40351: Network error: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351, [1920ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.253:45437: Network error: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [1921ms] Sub RPC ConnectToMaster: received response from server master-127.7.143.252:41589: OK, [1921ms] delaying RPC due to: Service unavailable: Master config (127.7.143.252:41589,127.7.143.254:40351,127.7.143.253:45437) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437, [2057ms] refreshing cache from master, [2057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.252:41589, [2058ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.254:40351, [2059ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.143.253:45437, [2061ms] too many traces: truncated at 100 traces, deferred=Deferred@1619225268(state=PENDING, result=null, callback=(continuation of Deferred@1546163120 after retry RPC after error@360121393) -> (continuation of Deferred@1164840193 after retry RPC after error@1044584914) -> (continuation of Deferred@644786361 after retry RPC after error@185956041) -> (continuation of Deferred@1042707631 after retry RPC after error@403535664) -> (continuation of Deferred@1265971176 after retry RPC after error@1802939249) -> (continuation of Deferred@1727539644 after retry RPC after error@364865188) -> (continuation of Deferred@2010583444 after retry RPC after error@690225468) -> (continuation of Deferred@238450797 after retry RPC after error@1624417609) -> (continuation of Deferred@801526646 after retry RPC after error@652380755) -> (continuation of Deferred@455769074 after retry RPC after error@1856470323) -> (continuation of Deferred@1936186994 after retry RPC after error@1097167455) -> (continuation of Deferred@156791914 after retry RPC after error@1556340375) -> (continuation of Deferred@286924595 after retry RPC after error@2064085189) -> (continuation of Deferred@825663723 after retry RPC after error@1942218556) -> (continuation of Deferred@845254128 after retry RPC after error@1923109409) -> (continuation of Deferred@1989514598 after retry RPC after error@436430479) -> (continuation of Deferred@607348274 after retry RPC after error@1416759759) -> (continuation of Deferred@244750466 after retry RPC after error@262185153) -> (continuation of Deferred@807224817 after retry RPC after error@1961168460) -> (continuation of Deferred@1317915354 after retry RPC after error@1151458658) -> (continuation of Deferred@508111562 after retry RPC after error@1107476513) -> (continuation of Deferred@1652099489 after retry RPC after error@679055937) -> (continuation of Deferred@2142364414 after retry RPC after error@736503521) -> (continuation of Deferred@33035837 after retry RPC after error@1702324185) -> (continuation of Deferred@354541467 after retry RPC after error@267006704) -> (continuation of Deferred@1315083470 after retry RPC after error@284523380) -> (continuation of Deferred@196326390 after retry RPC after error@1427144130), errback=(continuation of Deferred@1546163120 after retry RPC after error@360121393) -> (continuation of Deferred@1164840193 after retry RPC after error@1044584914) -> (continuation of Deferred@644786361 after retry RPC after error@185956041) -> (continuation of Deferred@1042707631 after retry RPC after error@403535664) -> (continuation of Deferred@1265971176 after retry RPC after error@1802939249) -> (continuation of Deferred@1727539644 after retry RPC after error@364865188) -> (continuation of Deferred@2010583444 after retry RPC after error@690225468) -> (continuation of Deferred@238450797 after retry RPC after error@1624417609) -> (continuation of Deferred@801526646 after retry RPC after error@652380755) -> (continuation of Deferred@455769074 after retry RPC after error@1856470323) -> (continuation of Deferred@1936186994 after retry RPC after error@1097167455) -> (continuation of Deferred@156791914 after retry RPC after error@1556340375) -> (continuation of Deferred@286924595 after retry RPC after error@2064085189) -> (continuation of Deferred@825663723 after retry RPC after error@1942218556) -> (continuation of Deferred@845254128 after retry RPC after error@1923109409) -> (continuation of Deferred@1989514598 after retry RPC after error@436430479) -> (continuation of Deferred@607348274 after retry RPC after error@1416759759) -> (continuation of Deferred@244750466 after retry RPC after error@262185153) -> (continuation of Deferred@807224817 after retry RPC after error@1961168460) -> (continuation of Deferred@1317915354 after retry RPC after error@1151458658) -> (continuation of Deferred@508111562 after retry RPC after error@1107476513) -> (continuation of Deferred@1652099489 after retry RPC after error@679055937) -> (continuation of Deferred@2142364414 after retry RPC after error@736503521) -> (continuation of Deferred@33035837 after retry RPC after error@1702324185) -> (continuation of Deferred@354541467 after retry RPC after error@267006704) -> (continuation of Deferred@1315083470 after retry RPC after error@284523380) -> (continuation of Deferred@196326390 after retry RPC after error@1427144130)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.7.143.252:41589,127.7.143.254:40351,127.7.143.253:45437) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.254:40351(127.7.143.254:40351): Connection refused: /127.7.143.254:40351,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.143.253:45437(127.7.143.253:45437): Connection refused: /127.7.143.253:45437
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.7.143.253:45437
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)