06:19:33.976 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
06:19:33.989 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskwUNst7/test-tmp
06:19:33.991 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
06:19:33.995 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskwUNst7/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
06:19:34.272 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
06:19:34.415 [DEBUG - main] (MiniKuduCluster.java:195) Response:
06:19:34.416 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
06:19:34.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:34.419078 2220 mini_oidc.cc:152] Starting JWKS server
06:19:34.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:34.422868 2220 webserver.cc:460] Webserver started at http://127.0.0.1:40523/ using document root <none> and password file <none>
06:19:34.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:34.423359 2220 mini_oidc.cc:189] Starting OIDC Discovery server
06:19:34.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:34.425324 2220 webserver.cc:460] Webserver started at http://127.0.0.1:45957/ using document root <none> and password file <none>
06:19:34.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T06:19:34Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
06:19:34.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T06:19:34Z Disabled control of system clock
06:19:34.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:34.492755 2220 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskwUNst7/build/tsan/bin/kudu
06:19:34.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskwUNst7/build/tsan/bin/kudu
06:19:34.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/master-0/wal
06:19:34.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/master-0/data
06:19:34.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:19:34.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:19:34.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:19:34.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:19:34.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:19:34.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:19:34.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/master-0/logs
06:19:34.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/master-0/data/info.pb
06:19:34.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:19:34.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:19:34.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:19:34.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:19:34.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:19:34.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:19:34.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:19:34.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:19:34.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
06:19:34.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:19:34.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:19:34.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:19:34.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.43.62:41039
06:19:34.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.43.62
06:19:34.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:19:34.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.43.20:38835
06:19:34.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:19:34.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:19:34.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:19:34.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:19:34.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.43.62:41039,127.2.43.61:45135,127.2.43.60:45273
06:19:34.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:19:34.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40523/jwks.json with env {}
06:19:34.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:34.781883 2232 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:19:34.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:34.782388 2232 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:19:34.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:34.782965 2232 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:19:34.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:34.811990 2232 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
06:19:34.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:34.812266 2232 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:19:34.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:34.812465 2232 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
06:19:34.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:34.812733 2232 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
06:19:34.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:34.813012 2232 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
06:19:34.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:34.813226 2232 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40523/jwks.json
06:19:34.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:34.845664 2232 master_runner.cc:386] Master server non-default flags:
06:19:34.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:19:34.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.43.20:38835
06:19:34.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:19:34.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:19:34.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/master-0/data
06:19:34.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/master-0/wal
06:19:34.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:19:34.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.43.62:41039,127.2.43.61:45135,127.2.43.60:45273
06:19:34.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:19:34.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:19:34.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:19:34.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.43.62:41039
06:19:34.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:19:34.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:19:34.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:19:34.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40523/jwks.json
06:19:34.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:19:34.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:19:34.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/master-0/data/info.pb
06:19:34.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.43.62
06:19:34.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:19:34.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:19:34.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:19:34.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:19:34.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:19:34.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:19:34.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/master-0/logs
06:19:34.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:19:34.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:19:34.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:19:34.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
06:19:34.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:19:34.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:19:34.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:19:34.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 05:19:33 UTC on ccycloud.quasar-ebyiam.root.comops.site
06:19:34.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4675663
06:19:34.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:19:34.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:34.847067 2232 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:19:34.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:34.849325 2232 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:19:34.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:34.866976 2238 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:19:36.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:34.867033 2239 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:19:36.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:34.867040 2241 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:19:36.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:34.869436 2232 server_base.cc:1034] running on GCE node
06:19:36.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:36.015658 2232 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:19:36.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:36.018347 2232 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:19:36.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:36.019662 2232 hybrid_clock.cc:648] HybridClock initialized: now 1790489976019616 us; error 68 us; skew 500 ppm
06:19:36.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:36.020498 2232 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:19:36.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:36.032719 2232 webserver.cc:460] Webserver started at http://127.2.43.62:37209/ using document root <none> and password file <none>
06:19:36.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:36.033607 2232 fs_manager.cc:359] Metadata directory not provided
06:19:36.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:36.033792 2232 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:19:36.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:36.034183 2232 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:19:36.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:36.038718 2232 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/master-0/data/instance:
06:19:36.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "19fa63f7ad554a74877915078159d1f5"
06:19:36.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 06:19:36 on dist-test-slave-0tdn"
06:19:36.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:36.039908 2232 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/master-0/wal/instance:
06:19:36.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "19fa63f7ad554a74877915078159d1f5"
06:19:36.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 06:19:36 on dist-test-slave-0tdn"
06:19:36.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:36.047549 2232 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
06:19:36.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:36.052726 2248 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:19:36.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:36.053658 2232 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.002s
06:19:36.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:36.054030 2232 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/master-0/data,/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/master-0/wal
06:19:36.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "19fa63f7ad554a74877915078159d1f5"
06:19:36.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 06:19:36 on dist-test-slave-0tdn"
06:19:36.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:36.054409 2232 fs_report.cc:383] FS layout report
06:19:36.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:19:36.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/master-0/wal
06:19:36.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/master-0/wal
06:19:36.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/master-0/data/data
06:19:36.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:19:36.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:19:36.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:19:36.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:19:36.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:19:36.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:19:36.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:19:36.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:19:36.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:19:36.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:19:36.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:19:36.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:36.120875 2232 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
06:19:36.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:36.122426 2232 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:19:36.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:36.122946 2232 kserver.cc:163] Server-wide thread pool size limit: 3276
06:19:36.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:36.197474 2232 rpc_server.cc:307] RPC server started. Bound to: 127.2.43.62:41039
06:19:36.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:36.197567 2299 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.2.43.62:41039 every 8 connection(s)
06:19:36.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:36.200410 2232 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/master-0/data/info.pb
06:19:36.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:36.202853 2220 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskwUNst7/build/tsan/bin/kudu as pid 2232
06:19:36.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:36.203421 2220 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/master-0/wal/instance
06:19:36.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:36.206954 2300 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
06:19:36.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:36.207650 2220 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskwUNst7/build/tsan/bin/kudu
06:19:36.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskwUNst7/build/tsan/bin/kudu
06:19:36.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/master-1/wal
06:19:36.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/master-1/data
06:19:36.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:19:36.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:19:36.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:19:36.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:19:36.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:19:36.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:19:36.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/master-1/logs
06:19:36.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/master-1/data/info.pb
06:19:36.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:19:36.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:19:36.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:19:36.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:19:36.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:19:36.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:19:36.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:19:36.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:19:36.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
06:19:36.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:19:36.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:19:36.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:19:36.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.43.61:45135
06:19:36.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.43.61
06:19:36.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:19:36.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.43.20:38835
06:19:36.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:19:36.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:19:36.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:19:36.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:19:36.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.43.62:41039,127.2.43.61:45135,127.2.43.60:45273
06:19:36.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:19:36.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40523/jwks.json with env {}
06:19:36.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:36.220554 2300 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.43.62" port: 41039 } has no permanent_uuid. Determining permanent_uuid...
06:19:36.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:36.251402 2300 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.43.61" port: 45135 } has no permanent_uuid. Determining permanent_uuid...
06:19:36.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:36.253748 2251 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.43.61:45135: connect: Connection refused (error 111)
06:19:36.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:36.258152 2300 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.43.61:45135: Network error: Client connection negotiation failed: client connection to 127.2.43.61:45135: connect: Connection refused (error 111)
06:19:36.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:36.307619 2300 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.43.61" port: 45135 } attempt: 1
06:19:36.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:36.311422 2300 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.43.61:45135: Network error: Client connection negotiation failed: client connection to 127.2.43.61:45135: connect: Connection refused (error 111)
06:19:36.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:36.379801 2300 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.43.61" port: 45135 } attempt: 2
06:19:36.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:36.383154 2300 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.43.61:45135: Network error: Client connection negotiation failed: client connection to 127.2.43.61:45135: connect: Connection refused (error 111)
06:19:36.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:36.474548 2300 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.43.61" port: 45135 } attempt: 3
06:19:36.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:36.478111 2300 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.43.61:45135: Network error: Client connection negotiation failed: client connection to 127.2.43.61:45135: connect: Connection refused (error 111)
06:19:36.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:36.521083 2302 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:19:36.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:36.521613 2302 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:19:36.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:36.522123 2302 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:19:36.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:36.551281 2302 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
06:19:36.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:36.551594 2302 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:19:36.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:36.551847 2302 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
06:19:36.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:36.552042 2302 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
06:19:36.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:36.552248 2302 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
06:19:36.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:36.552413 2302 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40523/jwks.json
06:19:36.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:36.584952 2302 master_runner.cc:386] Master server non-default flags:
06:19:36.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:19:36.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.43.20:38835
06:19:36.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:19:36.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:19:36.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/master-1/data
06:19:36.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/master-1/wal
06:19:36.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:19:36.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.43.62:41039,127.2.43.61:45135,127.2.43.60:45273
06:19:36.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:19:36.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:19:36.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:19:36.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.43.61:45135
06:19:36.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:19:36.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:19:36.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:19:36.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40523/jwks.json
06:19:36.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:19:36.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:19:36.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/master-1/data/info.pb
06:19:36.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.43.61
06:19:36.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:19:36.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:19:36.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:19:36.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:19:36.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:19:36.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:19:36.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/master-1/logs
06:19:36.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:19:36.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:19:36.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:19:36.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
06:19:36.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:19:36.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:19:36.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:19:36.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 05:19:33 UTC on ccycloud.quasar-ebyiam.root.comops.site
06:19:36.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4675663
06:19:36.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:19:36.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:36.586222 2302 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:19:36.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:36.587934 2302 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:19:36.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:36.605331 2314 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:19:36.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:36.621507 2300 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.43.61" port: 45135 } attempt: 4
06:19:36.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:36.624936 2300 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.43.61:45135: Network error: Client connection negotiation failed: client connection to 127.2.43.61:45135: connect: Connection refused (error 111)
06:19:36.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:36.924579 2300 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.43.61" port: 45135 } attempt: 5
06:19:36.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:36.928362 2300 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.43.61:45135: Network error: Client connection negotiation failed: client connection to 127.2.43.61:45135: connect: Connection refused (error 111)
06:19:37.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:37.475958 2300 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.43.61" port: 45135 } attempt: 6
06:19:37.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:37.482821 2300 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.43.61:45135: Network error: Client connection negotiation failed: client connection to 127.2.43.61:45135: connect: Connection refused (error 111)
06:19:37.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:36.605396 2312 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:19:37.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:36.606935 2302 server_base.cc:1034] running on GCE node
06:19:37.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:36.607184 2311 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:19:37.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:37.755105 2302 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:19:37.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:37.758684 2302 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:19:37.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:37.760075 2302 hybrid_clock.cc:648] HybridClock initialized: now 1790489977760044 us; error 52 us; skew 500 ppm
06:19:37.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:37.761157 2302 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:19:37.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:37.767876 2302 webserver.cc:460] Webserver started at http://127.2.43.61:37547/ using document root <none> and password file <none>
06:19:37.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:37.769003 2302 fs_manager.cc:359] Metadata directory not provided
06:19:37.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:37.769302 2302 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:19:37.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:37.769876 2302 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:19:37.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:37.774852 2302 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/master-1/data/instance:
06:19:37.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "54060b4ec97245ebba56bca24b9aff37"
06:19:37.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 06:19:37 on dist-test-slave-0tdn"
06:19:37.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:37.776160 2302 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/master-1/wal/instance:
06:19:37.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "54060b4ec97245ebba56bca24b9aff37"
06:19:37.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 06:19:37 on dist-test-slave-0tdn"
06:19:37.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:37.783145 2302 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.006s sys 0.000s
06:19:37.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:37.788699 2323 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:19:37.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:37.789688 2302 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
06:19:37.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:37.789947 2302 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/master-1/data,/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/master-1/wal
06:19:37.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "54060b4ec97245ebba56bca24b9aff37"
06:19:37.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 06:19:37 on dist-test-slave-0tdn"
06:19:37.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:37.790215 2302 fs_report.cc:383] FS layout report
06:19:37.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:19:37.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/master-1/wal
06:19:37.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/master-1/wal
06:19:37.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/master-1/data/data
06:19:37.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:19:37.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:19:37.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:19:37.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:19:37.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:19:37.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:19:37.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:19:37.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:19:37.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:19:37.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:19:37.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:19:37.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:37.850163 2302 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
06:19:37.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:37.851550 2302 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:19:37.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:37.852327 2302 kserver.cc:163] Server-wide thread pool size limit: 3276
06:19:37.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:37.923851 2302 rpc_server.cc:307] RPC server started. Bound to: 127.2.43.61:45135
06:19:37.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:37.923910 2374 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.2.43.61:45135 every 8 connection(s)
06:19:37.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:37.926682 2302 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/master-1/data/info.pb
06:19:37.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:37.931654 2375 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
06:19:37.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:37.935920 2220 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskwUNst7/build/tsan/bin/kudu as pid 2302
06:19:37.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:37.936342 2220 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/master-1/wal/instance
06:19:37.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:37.938378 2220 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskwUNst7/build/tsan/bin/kudu
06:19:37.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskwUNst7/build/tsan/bin/kudu
06:19:37.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/master-2/wal
06:19:37.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/master-2/data
06:19:37.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:19:37.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:19:37.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:19:37.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:19:37.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:19:37.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:19:37.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/master-2/logs
06:19:37.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/master-2/data/info.pb
06:19:37.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:19:37.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:19:37.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:19:37.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:19:37.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:19:37.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:19:37.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:19:37.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:19:37.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
06:19:37.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:19:37.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:19:37.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:19:37.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.43.60:45273
06:19:37.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.43.60
06:19:37.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:19:37.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.43.20:38835
06:19:37.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:19:37.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:19:37.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:19:37.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:19:37.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.43.62:41039,127.2.43.61:45135,127.2.43.60:45273
06:19:37.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:19:37.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40523/jwks.json with env {}
06:19:37.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:37.943234 2375 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.43.62" port: 41039 } has no permanent_uuid. Determining permanent_uuid...
06:19:37.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:37.968254 2375 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.43.61" port: 45135 } has no permanent_uuid. Determining permanent_uuid...
06:19:37.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:37.982781 2375 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.43.60" port: 45273 } has no permanent_uuid. Determining permanent_uuid...
06:19:37.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:37.985247 2327 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.43.60:45273: connect: Connection refused (error 111)
06:19:37.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:37.989440 2375 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.43.60:45273: Network error: Client connection negotiation failed: client connection to 127.2.43.60:45273: connect: Connection refused (error 111)
06:19:38.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:38.038995 2375 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.43.60" port: 45273 } attempt: 1
06:19:38.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:38.042652 2375 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.43.60:45273: Network error: Client connection negotiation failed: client connection to 127.2.43.60:45273: connect: Connection refused (error 111)
06:19:38.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:38.111027 2375 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.43.60" port: 45273 } attempt: 2
06:19:38.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:38.114940 2375 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.43.60:45273: Network error: Client connection negotiation failed: client connection to 127.2.43.60:45273: connect: Connection refused (error 111)
06:19:38.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:38.206352 2375 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.43.60" port: 45273 } attempt: 3
06:19:38.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:38.211203 2375 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.43.60:45273: Network error: Client connection negotiation failed: client connection to 127.2.43.60:45273: connect: Connection refused (error 111)
06:19:38.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:38.246240 2377 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:19:38.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:38.246949 2377 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:19:38.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:38.247634 2377 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:19:38.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:38.277978 2377 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
06:19:38.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:38.278333 2377 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:19:38.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:38.278647 2377 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
06:19:38.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:38.278959 2377 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
06:19:38.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:38.279289 2377 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
06:19:38.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:38.279575 2377 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40523/jwks.json
06:19:38.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:38.312245 2377 master_runner.cc:386] Master server non-default flags:
06:19:38.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:19:38.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.43.20:38835
06:19:38.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:19:38.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:19:38.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/master-2/data
06:19:38.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/master-2/wal
06:19:38.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:19:38.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.43.62:41039,127.2.43.61:45135,127.2.43.60:45273
06:19:38.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:19:38.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:19:38.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:19:38.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.43.60:45273
06:19:38.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:19:38.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:19:38.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:19:38.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40523/jwks.json
06:19:38.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:19:38.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:19:38.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/master-2/data/info.pb
06:19:38.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.43.60
06:19:38.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:19:38.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:19:38.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:19:38.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:19:38.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:19:38.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:19:38.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/master-2/logs
06:19:38.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:19:38.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:19:38.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:19:38.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
06:19:38.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:19:38.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:19:38.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:19:38.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 05:19:33 UTC on ccycloud.quasar-ebyiam.root.comops.site
06:19:38.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4675663
06:19:38.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:19:38.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:38.313750 2377 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:19:38.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:38.315744 2377 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:19:38.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:38.335916 2390 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:19:38.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:38.354715 2375 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.43.60" port: 45273 } attempt: 4
06:19:38.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:38.359758 2375 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.43.60:45273: Network error: Client connection negotiation failed: client connection to 127.2.43.60:45273: connect: Connection refused (error 111)
06:19:38.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:38.543424 2300 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.43.61" port: 45135 } attempt: 7
06:19:38.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:38.557123 2300 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.43.60" port: 45273 } has no permanent_uuid. Determining permanent_uuid...
06:19:38.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:38.562096 2300 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.43.60:45273: Network error: Client connection negotiation failed: client connection to 127.2.43.60:45273: connect: Connection refused (error 111)
06:19:38.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:38.620465 2300 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.43.60" port: 45273 } attempt: 1
06:19:38.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:38.623919 2300 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.43.60:45273: Network error: Client connection negotiation failed: client connection to 127.2.43.60:45273: connect: Connection refused (error 111)
06:19:38.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:38.659176 2375 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.43.60" port: 45273 } attempt: 5
06:19:38.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:38.663287 2375 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.43.60:45273: Network error: Client connection negotiation failed: client connection to 127.2.43.60:45273: connect: Connection refused (error 111)
06:19:38.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:38.705538 2300 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.43.60" port: 45273 } attempt: 2
06:19:38.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:38.709848 2300 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.43.60:45273: Network error: Client connection negotiation failed: client connection to 127.2.43.60:45273: connect: Connection refused (error 111)
06:19:38.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:38.795439 2300 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.43.60" port: 45273 } attempt: 3
06:19:38.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:38.799031 2300 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.43.60:45273: Network error: Client connection negotiation failed: client connection to 127.2.43.60:45273: connect: Connection refused (error 111)
06:19:38.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:38.939574 2300 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.43.60" port: 45273 } attempt: 4
06:19:38.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:38.944221 2300 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.43.60:45273: Network error: Client connection negotiation failed: client connection to 127.2.43.60:45273: connect: Connection refused (error 111)
06:19:39.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.210708 2375 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.43.60" port: 45273 } attempt: 6
06:19:39.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:39.218014 2375 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.43.60:45273: Network error: Client connection negotiation failed: client connection to 127.2.43.60:45273: connect: Connection refused (error 111)
06:19:39.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.227622 2300 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.43.60" port: 45273 } attempt: 5
06:19:39.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:39.231439 2300 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.43.60:45273: Network error: Client connection negotiation failed: client connection to 127.2.43.60:45273: connect: Connection refused (error 111)
06:19:39.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:38.336193 2388 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:19:39.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:38.337680 2377 server_base.cc:1034] running on GCE node
06:19:39.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:38.337095 2387 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:19:39.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.483881 2377 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:19:39.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.486691 2377 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:19:39.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.488044 2377 hybrid_clock.cc:648] HybridClock initialized: now 1790489979488012 us; error 42 us; skew 500 ppm
06:19:39.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.488993 2377 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:19:39.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.495148 2377 webserver.cc:460] Webserver started at http://127.2.43.60:43871/ using document root <none> and password file <none>
06:19:39.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.496217 2377 fs_manager.cc:359] Metadata directory not provided
06:19:39.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.496449 2377 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:19:39.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.497009 2377 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:19:39.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.501627 2377 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/master-2/data/instance:
06:19:39.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0c1c70b81b304462b7e21f739ac90081"
06:19:39.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 06:19:39 on dist-test-slave-0tdn"
06:19:39.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.502959 2377 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/master-2/wal/instance:
06:19:39.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0c1c70b81b304462b7e21f739ac90081"
06:19:39.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 06:19:39 on dist-test-slave-0tdn"
06:19:39.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.510360 2377 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.004s
06:19:39.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.516454 2402 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:19:39.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.517463 2377 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
06:19:39.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.517710 2377 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/master-2/data,/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/master-2/wal
06:19:39.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0c1c70b81b304462b7e21f739ac90081"
06:19:39.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 06:19:39 on dist-test-slave-0tdn"
06:19:39.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.517980 2377 fs_report.cc:383] FS layout report
06:19:39.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:19:39.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/master-2/wal
06:19:39.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/master-2/wal
06:19:39.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/master-2/data/data
06:19:39.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:19:39.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:19:39.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:19:39.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:19:39.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:19:39.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:19:39.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:19:39.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:19:39.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:19:39.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:19:39.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:19:39.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.556682 2377 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
06:19:39.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.558018 2377 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:19:39.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.558804 2377 kserver.cc:163] Server-wide thread pool size limit: 3276
06:19:39.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.640110 2377 rpc_server.cc:307] RPC server started. Bound to: 127.2.43.60:45273
06:19:39.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.640179 2453 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.2.43.60:45273 every 8 connection(s)
06:19:39.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.642635 2377 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/master-2/data/info.pb
06:19:39.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.647689 2220 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskwUNst7/build/tsan/bin/kudu as pid 2377
06:19:39.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.647955 2454 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
06:19:39.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.648159 2220 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/master-2/wal/instance
06:19:39.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.666450 2454 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.43.62" port: 41039 } has no permanent_uuid. Determining permanent_uuid...
06:19:39.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.694044 2454 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.43.61" port: 45135 } has no permanent_uuid. Determining permanent_uuid...
06:19:39.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.705382 2454 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.43.60" port: 45273 } has no permanent_uuid. Determining permanent_uuid...
06:19:39.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.723650 2454 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0c1c70b81b304462b7e21f739ac90081: Bootstrap starting.
06:19:39.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.730193 2454 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 0c1c70b81b304462b7e21f739ac90081: Neither blocks nor log segments found. Creating new log.
06:19:39.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.731688 2454 log.cc:824] T 00000000000000000000000000000000 P 0c1c70b81b304462b7e21f739ac90081: Log is configured to *not* fsync() on all Append() calls
06:19:39.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.736807 2454 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0c1c70b81b304462b7e21f739ac90081: No bootstrap required, opened a new log
06:19:39.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.757436 2454 raft_consensus.cc:348] T 00000000000000000000000000000000 P 0c1c70b81b304462b7e21f739ac90081 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "19fa63f7ad554a74877915078159d1f5" member_type: VOTER last_known_addr { host: "127.2.43.62" port: 41039 } } peers { permanent_uuid: "54060b4ec97245ebba56bca24b9aff37" member_type: VOTER last_known_addr { host: "127.2.43.61" port: 45135 } } peers { permanent_uuid: "0c1c70b81b304462b7e21f739ac90081" member_type: VOTER last_known_addr { host: "127.2.43.60" port: 45273 } }
06:19:39.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.757970 2454 raft_consensus.cc:374] T 00000000000000000000000000000000 P 0c1c70b81b304462b7e21f739ac90081 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:19:39.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.758162 2454 raft_consensus.cc:729] T 00000000000000000000000000000000 P 0c1c70b81b304462b7e21f739ac90081 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0c1c70b81b304462b7e21f739ac90081, State: Initialized, Role: FOLLOWER
06:19:39.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.758823 2454 consensus_queue.cc:260] T 00000000000000000000000000000000 P 0c1c70b81b304462b7e21f739ac90081 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "19fa63f7ad554a74877915078159d1f5" member_type: VOTER last_known_addr { host: "127.2.43.62" port: 41039 } } peers { permanent_uuid: "54060b4ec97245ebba56bca24b9aff37" member_type: VOTER last_known_addr { host: "127.2.43.61" port: 45135 } } peers { permanent_uuid: "0c1c70b81b304462b7e21f739ac90081" member_type: VOTER last_known_addr { host: "127.2.43.60" port: 45273 } }
06:19:39.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.762182 2463 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0c1c70b81b304462b7e21f739ac90081 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "19fa63f7ad554a74877915078159d1f5" member_type: VOTER last_known_addr { host: "127.2.43.62" port: 41039 } } peers { permanent_uuid: "54060b4ec97245ebba56bca24b9aff37" member_type: VOTER last_known_addr { host: "127.2.43.61" port: 45135 } } peers { permanent_uuid: "0c1c70b81b304462b7e21f739ac90081" member_type: VOTER last_known_addr { host: "127.2.43.60" port: 45273 } } }
06:19:39.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.763342 2463 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0c1c70b81b304462b7e21f739ac90081 [sys.catalog]: This master's current role is: FOLLOWER
06:19:39.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.763893 2454 sys_catalog.cc:564] T 00000000000000000000000000000000 P 0c1c70b81b304462b7e21f739ac90081 [sys.catalog]: configured and running, proceeding with master startup.
06:19:39.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.784090 2300 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.43.60" port: 45273 } attempt: 6
06:19:39.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:39.793411 2474 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0c1c70b81b304462b7e21f739ac90081: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:19:39.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:39.794112 2474 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:19:39.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.807034 2300 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 19fa63f7ad554a74877915078159d1f5: Bootstrap starting.
06:19:39.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.812944 2300 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 19fa63f7ad554a74877915078159d1f5: Neither blocks nor log segments found. Creating new log.
06:19:39.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:39.814105 2408 tablet.cc:2369] T 00000000000000000000000000000000 P 0c1c70b81b304462b7e21f739ac90081: Can't schedule compaction. Clean time has not been advanced past its initial value.
06:19:39.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.815054 2463 raft_consensus.cc:482] T 00000000000000000000000000000000 P 0c1c70b81b304462b7e21f739ac90081 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:19:39.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.815519 2463 raft_consensus.cc:504] T 00000000000000000000000000000000 P 0c1c70b81b304462b7e21f739ac90081 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "19fa63f7ad554a74877915078159d1f5" member_type: VOTER last_known_addr { host: "127.2.43.62" port: 41039 } } peers { permanent_uuid: "54060b4ec97245ebba56bca24b9aff37" member_type: VOTER last_known_addr { host: "127.2.43.61" port: 45135 } } peers { permanent_uuid: "0c1c70b81b304462b7e21f739ac90081" member_type: VOTER last_known_addr { host: "127.2.43.60" port: 45273 } }
06:19:39.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.816917 2300 log.cc:824] T 00000000000000000000000000000000 P 19fa63f7ad554a74877915078159d1f5: Log is configured to *not* fsync() on all Append() calls
06:19:39.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.787514 2377 master_runner.cc:186] Error getting master registration for 127.2.43.62:41039: OK, instance_id { permanent_uuid: "19fa63f7ad554a74877915078159d1f5" instance_seqno: 1790489976176656 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
06:19:39.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.818866 2377 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:19:39.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.820161 2463 leader_election.cc:288] T 00000000000000000000000000000000 P 0c1c70b81b304462b7e21f739ac90081 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 19fa63f7ad554a74877915078159d1f5 (127.2.43.62:41039), 54060b4ec97245ebba56bca24b9aff37 (127.2.43.61:45135)
06:19:39.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.823537 2300 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 19fa63f7ad554a74877915078159d1f5: No bootstrap required, opened a new log
06:19:39.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.848872 2300 raft_consensus.cc:348] T 00000000000000000000000000000000 P 19fa63f7ad554a74877915078159d1f5 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "19fa63f7ad554a74877915078159d1f5" member_type: VOTER last_known_addr { host: "127.2.43.62" port: 41039 } } peers { permanent_uuid: "54060b4ec97245ebba56bca24b9aff37" member_type: VOTER last_known_addr { host: "127.2.43.61" port: 45135 } } peers { permanent_uuid: "0c1c70b81b304462b7e21f739ac90081" member_type: VOTER last_known_addr { host: "127.2.43.60" port: 45273 } }
06:19:39.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.849488 2300 raft_consensus.cc:374] T 00000000000000000000000000000000 P 19fa63f7ad554a74877915078159d1f5 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:19:39.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.849766 2300 raft_consensus.cc:729] T 00000000000000000000000000000000 P 19fa63f7ad554a74877915078159d1f5 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 19fa63f7ad554a74877915078159d1f5, State: Initialized, Role: FOLLOWER
06:19:39.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.850660 2300 consensus_queue.cc:260] T 00000000000000000000000000000000 P 19fa63f7ad554a74877915078159d1f5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "19fa63f7ad554a74877915078159d1f5" member_type: VOTER last_known_addr { host: "127.2.43.62" port: 41039 } } peers { permanent_uuid: "54060b4ec97245ebba56bca24b9aff37" member_type: VOTER last_known_addr { host: "127.2.43.61" port: 45135 } } peers { permanent_uuid: "0c1c70b81b304462b7e21f739ac90081" member_type: VOTER last_known_addr { host: "127.2.43.60" port: 45273 } }
06:19:39.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.853667 2478 sys_catalog.cc:455] T 00000000000000000000000000000000 P 19fa63f7ad554a74877915078159d1f5 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "19fa63f7ad554a74877915078159d1f5" member_type: VOTER last_known_addr { host: "127.2.43.62" port: 41039 } } peers { permanent_uuid: "54060b4ec97245ebba56bca24b9aff37" member_type: VOTER last_known_addr { host: "127.2.43.61" port: 45135 } } peers { permanent_uuid: "0c1c70b81b304462b7e21f739ac90081" member_type: VOTER last_known_addr { host: "127.2.43.60" port: 45273 } } }
06:19:39.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.854609 2478 sys_catalog.cc:458] T 00000000000000000000000000000000 P 19fa63f7ad554a74877915078159d1f5 [sys.catalog]: This master's current role is: FOLLOWER
06:19:39.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.855383 2300 sys_catalog.cc:564] T 00000000000000000000000000000000 P 19fa63f7ad554a74877915078159d1f5 [sys.catalog]: configured and running, proceeding with master startup.
06:19:39.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.856015 2275 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0c1c70b81b304462b7e21f739ac90081" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "19fa63f7ad554a74877915078159d1f5" is_pre_election: true
06:19:39.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.856839 2275 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 19fa63f7ad554a74877915078159d1f5 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0c1c70b81b304462b7e21f739ac90081 in term 0.
06:19:39.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.858086 2403 leader_election.cc:302] T 00000000000000000000000000000000 P 0c1c70b81b304462b7e21f739ac90081 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0c1c70b81b304462b7e21f739ac90081, 19fa63f7ad554a74877915078159d1f5; no voters:
06:19:39.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.859663 2463 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 0c1c70b81b304462b7e21f739ac90081 [term 0 FOLLOWER]: Leader pre-election won for term 1
06:19:39.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.860015 2463 raft_consensus.cc:482] T 00000000000000000000000000000000 P 0c1c70b81b304462b7e21f739ac90081 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
06:19:39.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.860288 2463 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 0c1c70b81b304462b7e21f739ac90081 [term 0 FOLLOWER]: Advancing to term 1
06:19:39.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.867120 2463 raft_consensus.cc:504] T 00000000000000000000000000000000 P 0c1c70b81b304462b7e21f739ac90081 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "19fa63f7ad554a74877915078159d1f5" member_type: VOTER last_known_addr { host: "127.2.43.62" port: 41039 } } peers { permanent_uuid: "54060b4ec97245ebba56bca24b9aff37" member_type: VOTER last_known_addr { host: "127.2.43.61" port: 45135 } } peers { permanent_uuid: "0c1c70b81b304462b7e21f739ac90081" member_type: VOTER last_known_addr { host: "127.2.43.60" port: 45273 } }
06:19:39.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.869771 2275 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0c1c70b81b304462b7e21f739ac90081" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "19fa63f7ad554a74877915078159d1f5"
06:19:39.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.870479 2275 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 19fa63f7ad554a74877915078159d1f5 [term 0 FOLLOWER]: Advancing to term 1
06:19:39.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.876000 2463 leader_election.cc:288] T 00000000000000000000000000000000 P 0c1c70b81b304462b7e21f739ac90081 [CANDIDATE]: Term 1 election: Requested vote from peers 19fa63f7ad554a74877915078159d1f5 (127.2.43.62:41039), 54060b4ec97245ebba56bca24b9aff37 (127.2.43.61:45135)
06:19:39.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.881328 2275 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 19fa63f7ad554a74877915078159d1f5 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0c1c70b81b304462b7e21f739ac90081 in term 1.
06:19:39.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:39.881528 2254 tablet.cc:2369] T 00000000000000000000000000000000 P 19fa63f7ad554a74877915078159d1f5: Can't schedule compaction. Clean time has not been advanced past its initial value.
06:19:39.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.884981 2403 leader_election.cc:302] T 00000000000000000000000000000000 P 0c1c70b81b304462b7e21f739ac90081 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0c1c70b81b304462b7e21f739ac90081, 19fa63f7ad554a74877915078159d1f5; no voters:
06:19:39.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.886359 2463 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 0c1c70b81b304462b7e21f739ac90081 [term 1 FOLLOWER]: Leader election won for term 1
06:19:39.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.903280 2463 raft_consensus.cc:686] T 00000000000000000000000000000000 P 0c1c70b81b304462b7e21f739ac90081 [term 1 LEADER]: Becoming Leader. State: Replica: 0c1c70b81b304462b7e21f739ac90081, State: Running, Role: LEADER
06:19:39.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.904517 2463 consensus_queue.cc:237] T 00000000000000000000000000000000 P 0c1c70b81b304462b7e21f739ac90081 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "19fa63f7ad554a74877915078159d1f5" member_type: VOTER last_known_addr { host: "127.2.43.62" port: 41039 } } peers { permanent_uuid: "54060b4ec97245ebba56bca24b9aff37" member_type: VOTER last_known_addr { host: "127.2.43.61" port: 45135 } } peers { permanent_uuid: "0c1c70b81b304462b7e21f739ac90081" member_type: VOTER last_known_addr { host: "127.2.43.60" port: 45273 } }
06:19:39.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:39.933308 2490 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 19fa63f7ad554a74877915078159d1f5: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:19:39.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:39.934185 2490 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:19:39.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.951025 2487 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0c1c70b81b304462b7e21f739ac90081 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 0c1c70b81b304462b7e21f739ac90081. Latest consensus state: current_term: 1 leader_uuid: "0c1c70b81b304462b7e21f739ac90081" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "19fa63f7ad554a74877915078159d1f5" member_type: VOTER last_known_addr { host: "127.2.43.62" port: 41039 } } peers { permanent_uuid: "54060b4ec97245ebba56bca24b9aff37" member_type: VOTER last_known_addr { host: "127.2.43.61" port: 45135 } } peers { permanent_uuid: "0c1c70b81b304462b7e21f739ac90081" member_type: VOTER last_known_addr { host: "127.2.43.60" port: 45273 } } }
06:19:39.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.910202 2232 master_runner.cc:186] Error getting master registration for 127.2.43.61:45135: OK, instance_id { permanent_uuid: "54060b4ec97245ebba56bca24b9aff37" instance_seqno: 1790489977903945 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
06:19:39.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.953639 2487 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0c1c70b81b304462b7e21f739ac90081 [sys.catalog]: This master's current role is: LEADER
06:19:39.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.954155 2232 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:19:39.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.958631 2493 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
06:19:39.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:39.966347 2493 catalog_manager.cc:1484] Initializing Kudu cluster ID...
06:19:40.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:40.001431 2275 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 19fa63f7ad554a74877915078159d1f5 [term 1 FOLLOWER]: Refusing update from remote peer 0c1c70b81b304462b7e21f739ac90081: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
06:19:40.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:40.006893 2463 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 0c1c70b81b304462b7e21f739ac90081 [LEADER]: Connected to new peer: Peer: permanent_uuid: "19fa63f7ad554a74877915078159d1f5" member_type: VOTER last_known_addr { host: "127.2.43.62" port: 41039 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
06:19:40.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:40.052834 2478 sys_catalog.cc:455] T 00000000000000000000000000000000 P 19fa63f7ad554a74877915078159d1f5 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 0c1c70b81b304462b7e21f739ac90081. Latest consensus state: current_term: 1 leader_uuid: "0c1c70b81b304462b7e21f739ac90081" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "19fa63f7ad554a74877915078159d1f5" member_type: VOTER last_known_addr { host: "127.2.43.62" port: 41039 } } peers { permanent_uuid: "54060b4ec97245ebba56bca24b9aff37" member_type: VOTER last_known_addr { host: "127.2.43.61" port: 45135 } } peers { permanent_uuid: "0c1c70b81b304462b7e21f739ac90081" member_type: VOTER last_known_addr { host: "127.2.43.60" port: 45273 } } }
06:19:40.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:40.054854 2478 sys_catalog.cc:458] T 00000000000000000000000000000000 P 19fa63f7ad554a74877915078159d1f5 [sys.catalog]: This master's current role is: FOLLOWER
06:19:40.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:40.088308 2463 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0c1c70b81b304462b7e21f739ac90081 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "0c1c70b81b304462b7e21f739ac90081" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "19fa63f7ad554a74877915078159d1f5" member_type: VOTER last_known_addr { host: "127.2.43.62" port: 41039 } } peers { permanent_uuid: "54060b4ec97245ebba56bca24b9aff37" member_type: VOTER last_known_addr { host: "127.2.43.61" port: 45135 } } peers { permanent_uuid: "0c1c70b81b304462b7e21f739ac90081" member_type: VOTER last_known_addr { host: "127.2.43.60" port: 45273 } } }
06:19:40.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:40.089105 2463 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0c1c70b81b304462b7e21f739ac90081 [sys.catalog]: This master's current role is: LEADER
06:19:40.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:40.097536 2493 catalog_manager.cc:1347] Generated new cluster ID: 898f8ab10f5b4fc8a77044f978a4d0de
06:19:40.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:40.097807 2493 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
06:19:40.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:40.100014 2478 sys_catalog.cc:455] T 00000000000000000000000000000000 P 19fa63f7ad554a74877915078159d1f5 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "0c1c70b81b304462b7e21f739ac90081" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "19fa63f7ad554a74877915078159d1f5" member_type: VOTER last_known_addr { host: "127.2.43.62" port: 41039 } } peers { permanent_uuid: "54060b4ec97245ebba56bca24b9aff37" member_type: VOTER last_known_addr { host: "127.2.43.61" port: 45135 } } peers { permanent_uuid: "0c1c70b81b304462b7e21f739ac90081" member_type: VOTER last_known_addr { host: "127.2.43.60" port: 45273 } } }
06:19:40.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:40.100713 2478 sys_catalog.cc:458] T 00000000000000000000000000000000 P 19fa63f7ad554a74877915078159d1f5 [sys.catalog]: This master's current role is: FOLLOWER
06:19:40.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:40.130832 2493 catalog_manager.cc:1370] Generated new certificate authority record
06:19:40.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:40.133723 2493 catalog_manager.cc:1504] Loading token signing keys...
06:19:40.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:40.160909 2493 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 0c1c70b81b304462b7e21f739ac90081: Generated new TSK 0
06:19:40.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:40.162388 2493 catalog_manager.cc:1514] Initializing in-progress tserver states...
06:19:40.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:40.187589 2220 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskwUNst7/build/tsan/bin/kudu
06:19:40.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskwUNst7/build/tsan/bin/kudu
06:19:40.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/ts-0/wal
06:19:40.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/ts-0/data
06:19:40.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:19:40.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:19:40.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:19:40.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:19:40.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:19:40.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:19:40.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/ts-0/logs
06:19:40.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/ts-0/data/info.pb
06:19:40.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:19:40.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:19:40.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:19:40.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:19:40.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:19:40.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:19:40.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:19:40.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:19:40.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
06:19:40.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:19:40.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.43.1:0
06:19:40.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.2.43.1
06:19:40.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.43.1
06:19:40.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:19:40.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.2.43.62:41039,127.2.43.61:45135,127.2.43.60:45273
06:19:40.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.43.20:38835
06:19:40.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:19:40.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:19:40.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
06:19:40.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:40.278352 2375 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.43.60" port: 45273 } attempt: 7
06:19:40.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:40.325047 2375 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 54060b4ec97245ebba56bca24b9aff37: Bootstrap starting.
06:19:40.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:40.331957 2375 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 54060b4ec97245ebba56bca24b9aff37: Neither blocks nor log segments found. Creating new log.
06:19:40.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:40.335332 2375 log.cc:824] T 00000000000000000000000000000000 P 54060b4ec97245ebba56bca24b9aff37: Log is configured to *not* fsync() on all Append() calls
06:19:40.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:40.355693 2375 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 54060b4ec97245ebba56bca24b9aff37: No bootstrap required, opened a new log
06:19:40.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:40.416988 2375 raft_consensus.cc:348] T 00000000000000000000000000000000 P 54060b4ec97245ebba56bca24b9aff37 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "19fa63f7ad554a74877915078159d1f5" member_type: VOTER last_known_addr { host: "127.2.43.62" port: 41039 } } peers { permanent_uuid: "54060b4ec97245ebba56bca24b9aff37" member_type: VOTER last_known_addr { host: "127.2.43.61" port: 45135 } } peers { permanent_uuid: "0c1c70b81b304462b7e21f739ac90081" member_type: VOTER last_known_addr { host: "127.2.43.60" port: 45273 } }
06:19:40.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:40.417824 2375 raft_consensus.cc:374] T 00000000000000000000000000000000 P 54060b4ec97245ebba56bca24b9aff37 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:19:40.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:40.418089 2375 raft_consensus.cc:729] T 00000000000000000000000000000000 P 54060b4ec97245ebba56bca24b9aff37 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 54060b4ec97245ebba56bca24b9aff37, State: Initialized, Role: FOLLOWER
06:19:40.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:40.418998 2375 consensus_queue.cc:260] T 00000000000000000000000000000000 P 54060b4ec97245ebba56bca24b9aff37 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "19fa63f7ad554a74877915078159d1f5" member_type: VOTER last_known_addr { host: "127.2.43.62" port: 41039 } } peers { permanent_uuid: "54060b4ec97245ebba56bca24b9aff37" member_type: VOTER last_known_addr { host: "127.2.43.61" port: 45135 } } peers { permanent_uuid: "0c1c70b81b304462b7e21f739ac90081" member_type: VOTER last_known_addr { host: "127.2.43.60" port: 45273 } }
06:19:40.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:40.422956 2502 sys_catalog.cc:455] T 00000000000000000000000000000000 P 54060b4ec97245ebba56bca24b9aff37 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "19fa63f7ad554a74877915078159d1f5" member_type: VOTER last_known_addr { host: "127.2.43.62" port: 41039 } } peers { permanent_uuid: "54060b4ec97245ebba56bca24b9aff37" member_type: VOTER last_known_addr { host: "127.2.43.61" port: 45135 } } peers { permanent_uuid: "0c1c70b81b304462b7e21f739ac90081" member_type: VOTER last_known_addr { host: "127.2.43.60" port: 45273 } } }
06:19:40.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:40.425372 2502 sys_catalog.cc:458] T 00000000000000000000000000000000 P 54060b4ec97245ebba56bca24b9aff37 [sys.catalog]: This master's current role is: FOLLOWER
06:19:40.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:40.438088 2375 sys_catalog.cc:564] T 00000000000000000000000000000000 P 54060b4ec97245ebba56bca24b9aff37 [sys.catalog]: configured and running, proceeding with master startup.
06:19:40.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:40.438235 2349 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0c1c70b81b304462b7e21f739ac90081" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "54060b4ec97245ebba56bca24b9aff37"
06:19:40.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:40.438994 2349 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 54060b4ec97245ebba56bca24b9aff37 [term 0 FOLLOWER]: Advancing to term 1
06:19:40.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:40.441781 2350 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0c1c70b81b304462b7e21f739ac90081" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "54060b4ec97245ebba56bca24b9aff37" is_pre_election: true
06:19:40.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:40.450361 2349 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 54060b4ec97245ebba56bca24b9aff37 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0c1c70b81b304462b7e21f739ac90081 in term 1.
06:19:40.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:40.451676 2348 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 54060b4ec97245ebba56bca24b9aff37 [term 1 FOLLOWER]: Refusing update from remote peer 0c1c70b81b304462b7e21f739ac90081: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
06:19:40.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:40.460690 2487 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 0c1c70b81b304462b7e21f739ac90081 [LEADER]: Connected to new peer: Peer: permanent_uuid: "54060b4ec97245ebba56bca24b9aff37" member_type: VOTER last_known_addr { host: "127.2.43.61" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
06:19:40.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:40.525108 2502 sys_catalog.cc:455] T 00000000000000000000000000000000 P 54060b4ec97245ebba56bca24b9aff37 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 0c1c70b81b304462b7e21f739ac90081. Latest consensus state: current_term: 1 leader_uuid: "0c1c70b81b304462b7e21f739ac90081" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "19fa63f7ad554a74877915078159d1f5" member_type: VOTER last_known_addr { host: "127.2.43.62" port: 41039 } } peers { permanent_uuid: "54060b4ec97245ebba56bca24b9aff37" member_type: VOTER last_known_addr { host: "127.2.43.61" port: 45135 } } peers { permanent_uuid: "0c1c70b81b304462b7e21f739ac90081" member_type: VOTER last_known_addr { host: "127.2.43.60" port: 45273 } } }
06:19:40.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:40.527459 2502 sys_catalog.cc:458] T 00000000000000000000000000000000 P 54060b4ec97245ebba56bca24b9aff37 [sys.catalog]: This master's current role is: FOLLOWER
06:19:40.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:40.526633 2509 sys_catalog.cc:455] T 00000000000000000000000000000000 P 54060b4ec97245ebba56bca24b9aff37 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "0c1c70b81b304462b7e21f739ac90081" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "19fa63f7ad554a74877915078159d1f5" member_type: VOTER last_known_addr { host: "127.2.43.62" port: 41039 } } peers { permanent_uuid: "54060b4ec97245ebba56bca24b9aff37" member_type: VOTER last_known_addr { host: "127.2.43.61" port: 45135 } } peers { permanent_uuid: "0c1c70b81b304462b7e21f739ac90081" member_type: VOTER last_known_addr { host: "127.2.43.60" port: 45273 } } }
06:19:40.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:40.528198 2509 sys_catalog.cc:458] T 00000000000000000000000000000000 P 54060b4ec97245ebba56bca24b9aff37 [sys.catalog]: This master's current role is: FOLLOWER
06:19:40.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:40.561031 2518 catalog_manager.cc:1259] Loaded cluster ID: 898f8ab10f5b4fc8a77044f978a4d0de
06:19:40.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:40.564524 2518 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 54060b4ec97245ebba56bca24b9aff37: loading cluster ID for follower catalog manager: success
06:19:40.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:40.580647 2518 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 54060b4ec97245ebba56bca24b9aff37: acquiring CA information for follower catalog manager: success
06:19:40.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:40.594753 2518 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 54060b4ec97245ebba56bca24b9aff37: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
06:19:40.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:40.767028 2499 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:19:40.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:40.767745 2499 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:19:40.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:40.768517 2499 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:19:40.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:40.823364 2499 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:19:40.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:40.824756 2499 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.2.43.1
06:19:40.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:40.884444 2499 tablet_server_runner.cc:78] Tablet server non-default flags:
06:19:40.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:19:40.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.43.20:38835
06:19:40.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:19:40.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:19:40.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/ts-0/data
06:19:40.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/ts-0/wal
06:19:40.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:19:40.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:19:40.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.43.1:0
06:19:40.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:19:40.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:19:40.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:19:40.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/ts-0/data/info.pb
06:19:40.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.43.1
06:19:40.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:19:40.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.2.43.62:41039,127.2.43.61:45135,127.2.43.60:45273
06:19:40.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:19:40.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:19:40.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:19:40.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:19:40.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:19:40.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.2.43.1
06:19:40.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/ts-0/logs
06:19:40.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:19:40.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:19:40.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:19:40.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
06:19:40.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:19:40.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:19:40.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:19:40.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 05:19:33 UTC on ccycloud.quasar-ebyiam.root.comops.site
06:19:40.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4675663
06:19:40.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:19:40.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:40.886104 2499 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:19:40.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:40.888149 2499 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:19:40.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:40.932583 2526 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:19:40.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:40.939440 2490 catalog_manager.cc:1259] Loaded cluster ID: 898f8ab10f5b4fc8a77044f978a4d0de
06:19:40.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:40.941007 2490 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 19fa63f7ad554a74877915078159d1f5: loading cluster ID for follower catalog manager: success
06:19:40.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:40.965132 2490 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 19fa63f7ad554a74877915078159d1f5: acquiring CA information for follower catalog manager: success
06:19:40.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:40.971468 2490 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 19fa63f7ad554a74877915078159d1f5: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
06:19:41.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:41.082667 2487 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0c1c70b81b304462b7e21f739ac90081 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "0c1c70b81b304462b7e21f739ac90081" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "19fa63f7ad554a74877915078159d1f5" member_type: VOTER last_known_addr { host: "127.2.43.62" port: 41039 } } peers { permanent_uuid: "54060b4ec97245ebba56bca24b9aff37" member_type: VOTER last_known_addr { host: "127.2.43.61" port: 45135 } } peers { permanent_uuid: "0c1c70b81b304462b7e21f739ac90081" member_type: VOTER last_known_addr { host: "127.2.43.60" port: 45273 } } }
06:19:41.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:41.083458 2487 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0c1c70b81b304462b7e21f739ac90081 [sys.catalog]: This master's current role is: LEADER
06:19:42.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:42.310667 2524 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 2499
06:19:42.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:42.861452 2524 kernel_stack_watchdog.cc:198] Thread 2499 stuck at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:637 for 383ms:
06:19:42.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
06:19:42.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
06:19:42.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
06:19:42.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
06:19:42.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:40.933995 2525 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:19:42.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:42.862735 2499 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 1.938s user 0.636s sys 1.036s
06:19:42.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:42.863055 2499 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 1.938s user 0.636s sys 1.036s
06:19:42.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:42.863286 2499 server_base.cc:1034] running on GCE node
06:19:42.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:42.865082 2528 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:19:42.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:42.866721 2499 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:19:42.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:42.869539 2499 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:19:42.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:42.870978 2499 hybrid_clock.cc:648] HybridClock initialized: now 1790489982870931 us; error 45 us; skew 500 ppm
06:19:42.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:42.871876 2499 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:19:42.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:42.884995 2499 webserver.cc:460] Webserver started at http://127.2.43.1:37945/ using document root <none> and password file <none>
06:19:42.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:42.885977 2499 fs_manager.cc:359] Metadata directory not provided
06:19:42.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:42.886222 2499 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:19:42.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:42.886736 2499 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:19:42.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:42.893179 2499 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/ts-0/data/instance:
06:19:42.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ac5095582ae14294b662bdcfe5d63a40"
06:19:42.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 06:19:42 on dist-test-slave-0tdn"
06:19:42.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:42.894600 2499 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/ts-0/wal/instance:
06:19:42.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ac5095582ae14294b662bdcfe5d63a40"
06:19:42.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 06:19:42 on dist-test-slave-0tdn"
06:19:42.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:42.903718 2499 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.006s sys 0.004s
06:19:42.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:42.911303 2540 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:19:42.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:42.912483 2499 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
06:19:42.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:42.912814 2499 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/ts-0/data,/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/ts-0/wal
06:19:42.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ac5095582ae14294b662bdcfe5d63a40"
06:19:42.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 06:19:42 on dist-test-slave-0tdn"
06:19:42.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:42.913276 2499 fs_report.cc:383] FS layout report
06:19:42.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:19:42.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/ts-0/wal
06:19:42.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/ts-0/wal
06:19:42.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/ts-0/data/data
06:19:42.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:19:42.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:19:42.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:19:42.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:19:42.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:19:42.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:19:42.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:19:42.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:19:42.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:19:42.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:19:42.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:19:42.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:42.982720 2499 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
06:19:42.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:42.984612 2499 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:19:42.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:42.985152 2499 kserver.cc:163] Server-wide thread pool size limit: 3276
06:19:42.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:42.988824 2499 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
06:19:42.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:42.998505 2499 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
06:19:42.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:42.998780 2499 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
06:19:42.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:42.999040 2499 ts_tablet_manager.cc:610] Registered 0 tablets
06:19:42.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:42.999223 2499 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
06:19:43.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:43.220355 2499 rpc_server.cc:307] RPC server started. Bound to: 127.2.43.1:41627
06:19:43.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:43.220489 2652 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.2.43.1:41627 every 8 connection(s)
06:19:43.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:43.223737 2499 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/ts-0/data/info.pb
06:19:43.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:43.231165 2220 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskwUNst7/build/tsan/bin/kudu as pid 2499
06:19:43.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:43.231602 2220 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/ts-0/wal/instance
06:19:43.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:43.242687 2220 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskwUNst7/build/tsan/bin/kudu
06:19:43.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskwUNst7/build/tsan/bin/kudu
06:19:43.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/ts-1/wal
06:19:43.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/ts-1/data
06:19:43.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:19:43.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:19:43.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:19:43.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:19:43.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:19:43.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:19:43.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/ts-1/logs
06:19:43.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/ts-1/data/info.pb
06:19:43.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:19:43.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:19:43.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:19:43.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:19:43.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:19:43.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:19:43.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:19:43.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:19:43.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
06:19:43.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:19:43.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.43.2:0
06:19:43.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.2.43.2
06:19:43.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.43.2
06:19:43.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:19:43.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.2.43.62:41039,127.2.43.61:45135,127.2.43.60:45273
06:19:43.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.43.20:38835
06:19:43.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:19:43.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:19:43.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
06:19:43.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:43.323449 2654 heartbeater.cc:347] Connected to a master server at 127.2.43.60:45273
06:19:43.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:43.323989 2654 heartbeater.cc:464] Registering TS with master...
06:19:43.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:43.325569 2654 heartbeater.cc:511] Master 127.2.43.60:45273 requested a full tablet report, sending...
06:19:43.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:43.328547 2419 ts_manager.cc:194] Registered new tserver with Master: ac5095582ae14294b662bdcfe5d63a40 (127.2.43.1:41627)
06:19:43.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:43.329896 2653 heartbeater.cc:347] Connected to a master server at 127.2.43.61:45135
06:19:43.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:43.330227 2653 heartbeater.cc:464] Registering TS with master...
06:19:43.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:43.330978 2653 heartbeater.cc:511] Master 127.2.43.61:45135 requested a full tablet report, sending...
06:19:43.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:43.331252 2419 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.2.43.1:37985
06:19:43.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:43.332942 2658 heartbeater.cc:347] Connected to a master server at 127.2.43.62:41039
06:19:43.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:43.333220 2658 heartbeater.cc:464] Registering TS with master...
06:19:43.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:43.333930 2658 heartbeater.cc:511] Master 127.2.43.62:41039 requested a full tablet report, sending...
06:19:43.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:43.334108 2340 ts_manager.cc:194] Registered new tserver with Master: ac5095582ae14294b662bdcfe5d63a40 (127.2.43.1:41627)
06:19:43.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:43.338060 2265 ts_manager.cc:194] Registered new tserver with Master: ac5095582ae14294b662bdcfe5d63a40 (127.2.43.1:41627)
06:19:43.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:43.650780 2657 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:19:43.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:43.651506 2657 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:19:43.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:43.652709 2657 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:19:43.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:43.690236 2657 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:19:43.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:43.691208 2657 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.2.43.2
06:19:43.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:43.724248 2657 tablet_server_runner.cc:78] Tablet server non-default flags:
06:19:43.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:19:43.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.43.20:38835
06:19:43.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:19:43.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:19:43.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/ts-1/data
06:19:43.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/ts-1/wal
06:19:43.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:19:43.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:19:43.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.43.2:0
06:19:43.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:19:43.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:19:43.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:19:43.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/ts-1/data/info.pb
06:19:43.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.43.2
06:19:43.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:19:43.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.2.43.62:41039,127.2.43.61:45135,127.2.43.60:45273
06:19:43.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:19:43.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:19:43.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:19:43.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:19:43.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:19:43.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.2.43.2
06:19:43.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/ts-1/logs
06:19:43.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:19:43.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:19:43.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:19:43.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
06:19:43.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:19:43.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:19:43.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:19:43.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 05:19:33 UTC on ccycloud.quasar-ebyiam.root.comops.site
06:19:43.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4675663
06:19:43.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:19:43.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:43.725833 2657 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:19:43.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:43.727859 2657 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:19:43.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:43.749359 2669 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:19:44.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:44.337277 2654 heartbeater.cc:503] Master 127.2.43.60:45273 was elected leader, sending a full tablet report...
06:19:45.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:43.751684 2672 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:19:45.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:43.749608 2670 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:19:45.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:45.035218 2671 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1285 milliseconds
06:19:45.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:45.035380 2657 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
06:19:45.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:45.036826 2657 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:19:45.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:45.039353 2657 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:19:45.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:45.040735 2657 hybrid_clock.cc:648] HybridClock initialized: now 1790489985040710 us; error 40 us; skew 500 ppm
06:19:45.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:45.041435 2657 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:19:45.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:45.047655 2657 webserver.cc:460] Webserver started at http://127.2.43.2:45905/ using document root <none> and password file <none>
06:19:45.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:45.048522 2657 fs_manager.cc:359] Metadata directory not provided
06:19:45.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:45.048698 2657 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:19:45.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:45.049067 2657 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:19:45.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:45.053431 2657 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/ts-1/data/instance:
06:19:45.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c2ada5dec1a44138afa7946871b6844b"
06:19:45.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 06:19:45 on dist-test-slave-0tdn"
06:19:45.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:45.054401 2657 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/ts-1/wal/instance:
06:19:45.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c2ada5dec1a44138afa7946871b6844b"
06:19:45.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 06:19:45 on dist-test-slave-0tdn"
06:19:45.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:45.061055 2657 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.000s sys 0.008s
06:19:45.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:45.066551 2681 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:19:45.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:45.067498 2657 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
06:19:45.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:45.067860 2657 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/ts-1/data,/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/ts-1/wal
06:19:45.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c2ada5dec1a44138afa7946871b6844b"
06:19:45.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 06:19:45 on dist-test-slave-0tdn"
06:19:45.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:45.068198 2657 fs_report.cc:383] FS layout report
06:19:45.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:19:45.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/ts-1/wal
06:19:45.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/ts-1/wal
06:19:45.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/ts-1/data/data
06:19:45.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:19:45.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:19:45.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:19:45.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:19:45.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:19:45.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:19:45.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:19:45.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:19:45.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:19:45.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:19:45.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:19:45.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:45.101766 2657 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
06:19:45.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:45.103247 2657 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:19:45.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:45.103797 2657 kserver.cc:163] Server-wide thread pool size limit: 3276
06:19:45.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:45.106258 2657 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
06:19:45.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:45.110420 2657 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
06:19:45.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:45.110702 2657 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
06:19:45.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:45.111023 2657 ts_tablet_manager.cc:610] Registered 0 tablets
06:19:45.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:45.111238 2657 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
06:19:45.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:45.266348 2657 rpc_server.cc:307] RPC server started. Bound to: 127.2.43.2:41461
06:19:45.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:45.266465 2793 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.2.43.2:41461 every 8 connection(s)
06:19:45.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:45.269044 2657 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/ts-1/data/info.pb
06:19:45.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:45.276635 2220 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskwUNst7/build/tsan/bin/kudu as pid 2657
06:19:45.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:45.277293 2220 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/ts-1/wal/instance
06:19:45.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:45.292253 2220 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskwUNst7/build/tsan/bin/kudu
06:19:45.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskwUNst7/build/tsan/bin/kudu
06:19:45.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/ts-2/wal
06:19:45.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/ts-2/data
06:19:45.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:19:45.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:19:45.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:19:45.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:19:45.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:19:45.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:19:45.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/ts-2/logs
06:19:45.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/ts-2/data/info.pb
06:19:45.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:19:45.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:19:45.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:19:45.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:19:45.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:19:45.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:19:45.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:19:45.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:19:45.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
06:19:45.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:19:45.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.43.3:0
06:19:45.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.2.43.3
06:19:45.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.43.3
06:19:45.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:19:45.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.2.43.62:41039,127.2.43.61:45135,127.2.43.60:45273
06:19:45.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.43.20:38835
06:19:45.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:19:45.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:19:45.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
06:19:45.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:45.336236 2795 heartbeater.cc:347] Connected to a master server at 127.2.43.60:45273
06:19:45.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:45.336311 2794 heartbeater.cc:347] Connected to a master server at 127.2.43.61:45135
06:19:45.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:45.336230 2796 heartbeater.cc:347] Connected to a master server at 127.2.43.62:41039
06:19:45.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:45.336851 2795 heartbeater.cc:464] Registering TS with master...
06:19:45.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:45.336854 2796 heartbeater.cc:464] Registering TS with master...
06:19:45.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:45.337059 2794 heartbeater.cc:464] Registering TS with master...
06:19:45.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:45.338070 2796 heartbeater.cc:511] Master 127.2.43.62:41039 requested a full tablet report, sending...
06:19:45.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:45.338223 2794 heartbeater.cc:511] Master 127.2.43.61:45135 requested a full tablet report, sending...
06:19:45.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:45.338514 2795 heartbeater.cc:511] Master 127.2.43.60:45273 requested a full tablet report, sending...
06:19:45.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:45.340674 2419 ts_manager.cc:194] Registered new tserver with Master: c2ada5dec1a44138afa7946871b6844b (127.2.43.2:41461)
06:19:45.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:45.341145 2340 ts_manager.cc:194] Registered new tserver with Master: c2ada5dec1a44138afa7946871b6844b (127.2.43.2:41461)
06:19:45.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:45.342216 2265 ts_manager.cc:194] Registered new tserver with Master: c2ada5dec1a44138afa7946871b6844b (127.2.43.2:41461)
06:19:45.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:45.342984 2419 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.2.43.2:45407
06:19:45.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:45.625017 2803 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:19:45.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:45.625475 2803 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:19:45.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:45.625906 2803 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:19:45.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:45.673245 2803 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:19:45.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:45.674549 2803 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.2.43.3
06:19:45.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:45.708470 2803 tablet_server_runner.cc:78] Tablet server non-default flags:
06:19:45.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:19:45.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.43.20:38835
06:19:45.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:19:45.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:19:45.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/ts-2/data
06:19:45.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/ts-2/wal
06:19:45.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:19:45.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:19:45.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.43.3:0
06:19:45.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:19:45.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:19:45.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:19:45.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/ts-2/data/info.pb
06:19:45.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.43.3
06:19:45.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:19:45.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.2.43.62:41039,127.2.43.61:45135,127.2.43.60:45273
06:19:45.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:19:45.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:19:45.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:19:45.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:19:45.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:19:45.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.2.43.3
06:19:45.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/ts-2/logs
06:19:45.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:19:45.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:19:45.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:19:45.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
06:19:45.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:19:45.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:19:45.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:19:45.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 05:19:33 UTC on ccycloud.quasar-ebyiam.root.comops.site
06:19:45.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4675663
06:19:45.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:19:45.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:45.709616 2803 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:19:45.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:45.711405 2803 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:19:45.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:45.729171 2811 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:19:46.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:46.346251 2795 heartbeater.cc:503] Master 127.2.43.60:45273 was elected leader, sending a full tablet report...
06:19:46.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:45.729847 2813 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:19:46.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:45.729319 2810 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:19:46.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:46.891808 2812 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
06:19:46.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:46.891947 2803 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
06:19:46.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:46.896387 2803 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:19:46.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:46.898945 2803 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:19:46.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:46.900375 2803 hybrid_clock.cc:648] HybridClock initialized: now 1790489986900355 us; error 44 us; skew 500 ppm
06:19:46.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:46.901144 2803 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:19:46.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:46.907651 2803 webserver.cc:460] Webserver started at http://127.2.43.3:43263/ using document root <none> and password file <none>
06:19:46.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:46.908680 2803 fs_manager.cc:359] Metadata directory not provided
06:19:46.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:46.908854 2803 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:19:46.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:46.909265 2803 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:19:46.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:46.913796 2803 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/ts-2/data/instance:
06:19:46.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4aae07eb46e84dfcbf89af8047c3e1eb"
06:19:46.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 06:19:46 on dist-test-slave-0tdn"
06:19:46.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:46.914791 2803 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/ts-2/wal/instance:
06:19:46.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4aae07eb46e84dfcbf89af8047c3e1eb"
06:19:46.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 06:19:46 on dist-test-slave-0tdn"
06:19:46.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:46.921216 2803 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.005s sys 0.003s
06:19:46.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:46.927241 2821 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:19:46.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:46.928454 2803 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
06:19:46.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:46.928838 2803 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/ts-2/data,/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/ts-2/wal
06:19:46.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4aae07eb46e84dfcbf89af8047c3e1eb"
06:19:46.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 06:19:46 on dist-test-slave-0tdn"
06:19:46.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:46.929303 2803 fs_report.cc:383] FS layout report
06:19:46.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:19:46.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/ts-2/wal
06:19:46.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/ts-2/wal
06:19:46.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/ts-2/data/data
06:19:46.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:19:46.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:19:46.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:19:46.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:19:46.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:19:46.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:19:46.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:19:46.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:19:46.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:19:46.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:19:46.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:19:46.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:46.985069 2803 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
06:19:46.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:46.986415 2803 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:19:46.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:46.986931 2803 kserver.cc:163] Server-wide thread pool size limit: 3276
06:19:46.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:46.989485 2803 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
06:19:46.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:46.993345 2803 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
06:19:46.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:46.993567 2803 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
06:19:46.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:46.993875 2803 ts_tablet_manager.cc:610] Registered 0 tablets
06:19:46.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:46.994066 2803 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
06:19:47.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:47.138540 2803 rpc_server.cc:307] RPC server started. Bound to: 127.2.43.3:36589
06:19:47.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:47.138681 2933 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.2.43.3:36589 every 8 connection(s)
06:19:47.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:47.141124 2803 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/ts-2/data/info.pb
06:19:47.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:47.150317 2220 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskwUNst7/build/tsan/bin/kudu as pid 2803
06:19:47.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:47.150858 2220 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster2260052887791244488/ts-2/wal/instance
06:19:47.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:47.205633 2934 heartbeater.cc:347] Connected to a master server at 127.2.43.61:45135
06:19:47.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:47.206111 2934 heartbeater.cc:464] Registering TS with master...
06:19:47.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:47.207255 2934 heartbeater.cc:511] Master 127.2.43.61:45135 requested a full tablet report, sending...
06:19:47.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:47.207589 2935 heartbeater.cc:347] Connected to a master server at 127.2.43.60:45273
06:19:47.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:47.207882 2935 heartbeater.cc:464] Registering TS with master...
06:19:47.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:47.208623 2935 heartbeater.cc:511] Master 127.2.43.60:45273 requested a full tablet report, sending...
06:19:47.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:47.210652 2419 ts_manager.cc:194] Registered new tserver with Master: 4aae07eb46e84dfcbf89af8047c3e1eb (127.2.43.3:36589)
06:19:47.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:47.211865 2340 ts_manager.cc:194] Registered new tserver with Master: 4aae07eb46e84dfcbf89af8047c3e1eb (127.2.43.3:36589)
06:19:47.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:47.213038 2419 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.2.43.3:38831
06:19:47.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:47.226291 2936 heartbeater.cc:347] Connected to a master server at 127.2.43.62:41039
06:19:47.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:47.226558 2936 heartbeater.cc:464] Registering TS with master...
06:19:47.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:47.227022 2936 heartbeater.cc:511] Master 127.2.43.62:41039 requested a full tablet report, sending...
06:19:47.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:47.229568 2265 ts_manager.cc:194] Registered new tserver with Master: 4aae07eb46e84dfcbf89af8047c3e1eb (127.2.43.3:36589)
06:19:47.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:47.238471 2220 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
06:19:47.248 [DEBUG - main] (MiniKuduCluster.java:195) Response:
06:19:47.250 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
06:19:47.260 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.2.43.62"
port: 41039
}
bound_http_address {
host: "127.2.43.62"
port: 37209
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.2.43.61"
port: 45135
}
bound_http_address {
host: "127.2.43.61"
port: 37547
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.2.43.60"
port: 45273
}
bound_http_address {
host: "127.2.43.60"
port: 43871
}
}
}
06:19:47.267 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
06:19:47.270 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.2.43.1"
port: 41627
}
bound_http_address {
host: "127.2.43.1"
port: 37945
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.2.43.2"
port: 41461
}
bound_http_address {
host: "127.2.43.2"
port: 45905
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.2.43.3"
port: 36589
}
bound_http_address {
host: "127.2.43.3"
port: 43263
}
}
}
06:19:47.270 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
06:19:47.443 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
06:19:47.558 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskwUNst7/test-tmp
06:19:47.559 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
06:19:47.560 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskwUNst7/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
06:19:47.563 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
06:19:47.889 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
06:19:47.889 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
06:19:47.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:47.892647 2945 mini_oidc.cc:152] Starting JWKS server
06:19:47.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:47.896451 2945 webserver.cc:460] Webserver started at http://127.0.0.1:43029/ using document root <none> and password file <none>
06:19:47.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:47.896931 2945 mini_oidc.cc:189] Starting OIDC Discovery server
06:19:47.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:47.898470 2945 webserver.cc:460] Webserver started at http://127.0.0.1:37919/ using document root <none> and password file <none>
06:19:47.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
06:19:47.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/krb5kdc/principal' for realm 'KRBTEST.COM',
06:19:47.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
06:19:48.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 06:19:48 dist-test-slave-0tdn krb5kdc[2957](info): setting up network...
06:19:48.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
06:19:48.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 06:19:48 dist-test-slave-0tdn krb5kdc[2957](info): set up 2 sockets
06:19:48.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 06:19:48 dist-test-slave-0tdn krb5kdc[2957](info): commencing operation
06:19:48.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
06:19:48.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:48.216755 2935 heartbeater.cc:503] Master 127.2.43.60:45273 was elected leader, sending a full tablet report...
06:19:49.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:49.015597 2945 mini_kdc.cc:121] Time spent starting KDC: real 1.093s user 0.003s sys 0.018s
06:19:49.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
06:19:49.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
06:19:49.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
06:19:49.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
06:19:49.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
06:19:49.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
06:19:49.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
06:19:49.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
06:19:49.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
06:19:49.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
06:19:49.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/krb5kdc/test-user.keytab.
06:19:49.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/krb5kdc/test-user.keytab.
06:19:49.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 06:19:49 dist-test-slave-0tdn krb5kdc[2957](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790489989, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
06:19:49.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
06:19:49.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T06:19:49Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
06:19:49.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T06:19:49Z Disabled control of system clock
06:19:49.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.2.224.126@KRBTEST.COM; defaulting to no policy
06:19:49.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
06:19:49.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.2.224.126@KRBTEST.COM" created.
06:19:49.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
06:19:49.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.224.126 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/krb5kdc/kudu_127.2.224.126.keytab.
06:19:49.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.224.126 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/krb5kdc/kudu_127.2.224.126.keytab.
06:19:49.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:49.203493 2945 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskwUNst7/build/tsan/bin/kudu
06:19:49.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskwUNst7/build/tsan/bin/kudu
06:19:49.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-0/wal
06:19:49.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-0/data
06:19:49.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:19:49.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:19:49.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:19:49.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:19:49.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:19:49.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:19:49.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-0/logs
06:19:49.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-0/data/info.pb
06:19:49.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:19:49.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:19:49.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:19:49.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:19:49.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:19:49.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:19:49.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:19:49.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:19:49.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
06:19:49.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:19:49.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:19:49.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:19:49.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.224.126:38429
06:19:49.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.224.126
06:19:49.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:19:49.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/krb5kdc/kudu_127.2.224.126.keytab
06:19:49.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.224.126
06:19:49.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
06:19:49.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
06:19:49.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
06:19:49.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.224.84:37841
06:19:49.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:19:49.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:19:49.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:19:49.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:19:49.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.224.126:38429,127.2.224.125:37335,127.2.224.124:35387
06:19:49.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:19:49.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43029/jwks.json
06:19:49.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
06:19:49.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:49.503139 2971 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:19:49.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:49.503695 2971 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:19:49.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:49.504288 2971 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:19:49.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:49.533363 2971 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
06:19:49.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:49.533576 2971 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
06:19:49.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:49.533818 2971 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:19:49.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:49.533995 2971 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
06:19:49.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:49.534152 2971 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
06:19:49.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:49.534304 2971 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
06:19:49.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:49.534510 2971 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43029/jwks.json
06:19:49.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:49.570405 2971 master_runner.cc:386] Master server non-default flags:
06:19:49.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:19:49.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.224.84:37841
06:19:49.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:19:49.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:19:49.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-0/data
06:19:49.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-0/wal
06:19:49.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
06:19:49.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:19:49.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.224.126:38429,127.2.224.125:37335,127.2.224.124:35387
06:19:49.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/krb5kdc/kudu_127.2.224.126.keytab
06:19:49.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.224.126
06:19:49.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:19:49.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:19:49.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:19:49.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.224.126:38429
06:19:49.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:19:49.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:19:49.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:19:49.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43029/jwks.json
06:19:49.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
06:19:49.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
06:19:49.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
06:19:49.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:19:49.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:19:49.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-0/data/info.pb
06:19:49.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.224.126
06:19:49.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:19:49.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:19:49.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:19:49.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:19:49.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:19:49.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:19:49.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-0/logs
06:19:49.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:19:49.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:19:49.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:19:49.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
06:19:49.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:19:49.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:19:49.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:19:49.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 05:19:33 UTC on ccycloud.quasar-ebyiam.root.comops.site
06:19:49.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4675663
06:19:49.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:19:49.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:49.571573 2971 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:19:49.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:49.572685 2971 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:19:49.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:49.591836 2980 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:19:50.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:49.594029 2971 server_base.cc:1034] running on GCE node
06:19:50.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:49.591933 2977 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:19:50.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:49.591953 2978 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:19:50.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:50.790301 2971 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:19:50.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:50.792974 2971 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:19:50.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:50.794380 2971 hybrid_clock.cc:648] HybridClock initialized: now 1790489990794351 us; error 32 us; skew 500 ppm
06:19:50.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:50.795228 2971 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:19:50.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 06:19:50 dist-test-slave-0tdn krb5kdc[2957](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790489990, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.224.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
06:19:50.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:50.800892 2971 init.cc:377] Logged in from keytab as kudu/127.2.224.126@KRBTEST.COM (short username kudu)
06:19:50.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:50.805962 2971 webserver.cc:460] Webserver started at http://127.2.224.126:45581/ using document root <none> and password file <none>
06:19:50.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:50.806871 2971 fs_manager.cc:359] Metadata directory not provided
06:19:50.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:50.807103 2971 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:19:50.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:50.807593 2971 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:19:50.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:50.812327 2971 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-0/data/instance:
06:19:50.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8726f6ab2f024c2ea9d0e2c1ae24b2cd"
06:19:50.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 06:19:50 on dist-test-slave-0tdn"
06:19:50.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:50.813469 2971 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-0/wal/instance:
06:19:50.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8726f6ab2f024c2ea9d0e2c1ae24b2cd"
06:19:50.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 06:19:50 on dist-test-slave-0tdn"
06:19:50.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:50.820621 2971 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.007s sys 0.001s
06:19:50.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:50.825726 2990 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:19:50.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:50.826697 2971 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.001s
06:19:50.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:50.827064 2971 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-0/data,/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-0/wal
06:19:50.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8726f6ab2f024c2ea9d0e2c1ae24b2cd"
06:19:50.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 06:19:50 on dist-test-slave-0tdn"
06:19:50.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:50.827395 2971 fs_report.cc:383] FS layout report
06:19:50.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:19:50.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-0/wal
06:19:50.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-0/wal
06:19:50.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-0/data/data
06:19:50.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:19:50.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:19:50.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:19:50.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:19:50.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:19:50.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:19:50.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:19:50.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:19:50.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:19:50.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:19:50.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:19:50.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:50.866744 2971 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
06:19:50.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:50.871876 2971 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:19:50.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:50.872375 2971 kserver.cc:163] Server-wide thread pool size limit: 3276
06:19:50.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:50.948217 2971 rpc_server.cc:307] RPC server started. Bound to: 127.2.224.126:38429
06:19:50.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:50.948343 3041 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.2.224.126:38429 every 8 connection(s)
06:19:50.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:50.950817 2971 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-0/data/info.pb
06:19:50.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:50.951673 2945 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskwUNst7/build/tsan/bin/kudu as pid 2971
06:19:50.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:50.952690 2945 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-0/wal/instance
06:19:50.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:50.960196 3042 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
06:19:50.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.2.224.125@KRBTEST.COM; defaulting to no policy
06:19:50.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:50.972059 3042 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.224.126" port: 38429 } has no permanent_uuid. Determining permanent_uuid...
06:19:50.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
06:19:50.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.2.224.125@KRBTEST.COM" created.
06:19:51.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
06:19:51.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.224.125 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/krb5kdc/kudu_127.2.224.125.keytab.
06:19:51.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.224.125 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/krb5kdc/kudu_127.2.224.125.keytab.
06:19:51.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 06:19:51 dist-test-slave-0tdn krb5kdc[2957](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790489990, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.224.126@KRBTEST.COM for kudu/127.2.224.126@KRBTEST.COM
06:19:51.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:51.004900 2945 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskwUNst7/build/tsan/bin/kudu
06:19:51.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskwUNst7/build/tsan/bin/kudu
06:19:51.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-1/wal
06:19:51.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-1/data
06:19:51.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:19:51.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:19:51.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:19:51.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:19:51.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:19:51.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:19:51.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-1/logs
06:19:51.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-1/data/info.pb
06:19:51.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:19:51.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:19:51.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:19:51.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:19:51.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:19:51.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:19:51.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:19:51.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:19:51.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
06:19:51.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:19:51.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:19:51.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:19:51.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.224.125:37335
06:19:51.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.224.125
06:19:51.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:19:51.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/krb5kdc/kudu_127.2.224.125.keytab
06:19:51.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.224.125
06:19:51.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
06:19:51.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
06:19:51.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
06:19:51.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.224.84:37841
06:19:51.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:19:51.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:19:51.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:19:51.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:19:51.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.224.126:38429,127.2.224.125:37335,127.2.224.124:35387
06:19:51.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:19:51.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43029/jwks.json
06:19:51.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
06:19:51.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:51.018399 3042 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.224.125" port: 37335 } has no permanent_uuid. Determining permanent_uuid...
06:19:51.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:51.020990 2994 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.224.125:37335: connect: Connection refused (error 111)
06:19:51.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:51.024573 3042 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.224.125:37335: Network error: Client connection negotiation failed: client connection to 127.2.224.125:37335: connect: Connection refused (error 111)
06:19:51.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:51.073977 3042 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.224.125" port: 37335 } attempt: 1
06:19:51.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:51.078720 3042 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.224.125:37335: Network error: Client connection negotiation failed: client connection to 127.2.224.125:37335: connect: Connection refused (error 111)
06:19:51.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:51.147138 3042 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.224.125" port: 37335 } attempt: 2
06:19:51.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:51.150942 3042 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.224.125:37335: Network error: Client connection negotiation failed: client connection to 127.2.224.125:37335: connect: Connection refused (error 111)
06:19:51.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:51.242377 3042 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.224.125" port: 37335 } attempt: 3
06:19:51.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:51.246487 3042 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.224.125:37335: Network error: Client connection negotiation failed: client connection to 127.2.224.125:37335: connect: Connection refused (error 111)
06:19:51.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:51.319947 3048 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:19:51.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:51.320631 3048 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:19:51.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:51.321449 3048 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:19:51.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:51.353417 3048 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
06:19:51.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:51.353683 3048 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
06:19:51.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:51.354050 3048 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:19:51.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:51.354359 3048 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
06:19:51.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:51.354656 3048 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
06:19:51.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:51.354948 3048 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
06:19:51.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:51.355239 3048 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43029/jwks.json
06:19:51.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:51.389837 3042 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.224.125" port: 37335 } attempt: 4
06:19:51.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:51.393252 3042 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.224.125:37335: Network error: Client connection negotiation failed: client connection to 127.2.224.125:37335: connect: Connection refused (error 111)
06:19:51.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:51.396991 3048 master_runner.cc:386] Master server non-default flags:
06:19:51.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:19:51.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.224.84:37841
06:19:51.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:19:51.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:19:51.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-1/data
06:19:51.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-1/wal
06:19:51.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
06:19:51.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:19:51.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.224.126:38429,127.2.224.125:37335,127.2.224.124:35387
06:19:51.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/krb5kdc/kudu_127.2.224.125.keytab
06:19:51.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.224.125
06:19:51.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:19:51.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:19:51.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:19:51.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.224.125:37335
06:19:51.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:19:51.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:19:51.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:19:51.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43029/jwks.json
06:19:51.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
06:19:51.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
06:19:51.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
06:19:51.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:19:51.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:19:51.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-1/data/info.pb
06:19:51.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.224.125
06:19:51.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:19:51.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:19:51.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:19:51.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:19:51.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:19:51.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:19:51.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-1/logs
06:19:51.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:19:51.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:19:51.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:19:51.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
06:19:51.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:19:51.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:19:51.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:19:51.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 05:19:33 UTC on ccycloud.quasar-ebyiam.root.comops.site
06:19:51.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4675663
06:19:51.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:19:51.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:51.398263 3048 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:19:51.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:51.399667 3048 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:19:51.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:51.418390 3056 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:19:51.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:51.692653 3042 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.224.125" port: 37335 } attempt: 5
06:19:51.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:51.696699 3042 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.224.125:37335: Network error: Client connection negotiation failed: client connection to 127.2.224.125:37335: connect: Connection refused (error 111)
06:19:52.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:52.244061 3042 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.224.125" port: 37335 } attempt: 6
06:19:52.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:52.254978 3042 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.224.125:37335: Network error: Client connection negotiation failed: client connection to 127.2.224.125:37335: connect: Connection refused (error 111)
06:19:52.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:51.418967 3055 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:19:52.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:51.419065 3058 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:19:52.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:52.626060 3057 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
06:19:52.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:52.626264 3048 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
06:19:52.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:52.630156 3048 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:19:52.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:52.633419 3048 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:19:52.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:52.634815 3048 hybrid_clock.cc:648] HybridClock initialized: now 1790489992634788 us; error 51 us; skew 500 ppm
06:19:52.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:52.635673 3048 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:19:52.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 06:19:52 dist-test-slave-0tdn krb5kdc[2957](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790489992, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.224.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
06:19:52.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:52.641784 3048 init.cc:377] Logged in from keytab as kudu/127.2.224.125@KRBTEST.COM (short username kudu)
06:19:52.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:52.647507 3048 webserver.cc:460] Webserver started at http://127.2.224.125:34395/ using document root <none> and password file <none>
06:19:52.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:52.648437 3048 fs_manager.cc:359] Metadata directory not provided
06:19:52.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:52.648612 3048 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:19:52.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:52.649030 3048 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:19:52.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:52.653594 3048 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-1/data/instance:
06:19:52.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8a628d3d004b41b8beea7b87db236525"
06:19:52.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 06:19:52 on dist-test-slave-0tdn"
06:19:52.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:52.654594 3048 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-1/wal/instance:
06:19:52.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8a628d3d004b41b8beea7b87db236525"
06:19:52.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 06:19:52 on dist-test-slave-0tdn"
06:19:52.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:52.661955 3048 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.004s
06:19:52.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:52.667335 3069 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:19:52.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:52.668373 3048 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.001s sys 0.002s
06:19:52.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:52.668640 3048 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-1/data,/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-1/wal
06:19:52.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8a628d3d004b41b8beea7b87db236525"
06:19:52.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 06:19:52 on dist-test-slave-0tdn"
06:19:52.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:52.668900 3048 fs_report.cc:383] FS layout report
06:19:52.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:19:52.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-1/wal
06:19:52.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-1/wal
06:19:52.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-1/data/data
06:19:52.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:19:52.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:19:52.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:19:52.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:19:52.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:19:52.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:19:52.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:19:52.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:19:52.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:19:52.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:19:52.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:19:52.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:52.704571 3048 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
06:19:52.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:52.710279 3048 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:19:52.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:52.710664 3048 kserver.cc:163] Server-wide thread pool size limit: 3276
06:19:52.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:52.785871 3048 rpc_server.cc:307] RPC server started. Bound to: 127.2.224.125:37335
06:19:52.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:52.785972 3120 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.2.224.125:37335 every 8 connection(s)
06:19:52.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:52.788614 3048 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-1/data/info.pb
06:19:52.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:52.794623 3121 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
06:19:52.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:52.797129 2945 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskwUNst7/build/tsan/bin/kudu as pid 3048
06:19:52.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:52.797446 2945 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-1/wal/instance
06:19:52.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:52.805894 3121 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.224.126" port: 38429 } has no permanent_uuid. Determining permanent_uuid...
06:19:52.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.2.224.124@KRBTEST.COM; defaulting to no policy
06:19:52.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
06:19:52.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.2.224.124@KRBTEST.COM" created.
06:19:52.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 06:19:52 dist-test-slave-0tdn krb5kdc[2957](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790489992, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.224.125@KRBTEST.COM for kudu/127.2.224.126@KRBTEST.COM
06:19:52.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
06:19:52.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.224.124 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/krb5kdc/kudu_127.2.224.124.keytab.
06:19:52.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.224.124 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/krb5kdc/kudu_127.2.224.124.keytab.
06:19:52.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:52.840368 2945 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskwUNst7/build/tsan/bin/kudu
06:19:52.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskwUNst7/build/tsan/bin/kudu
06:19:52.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-2/wal
06:19:52.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-2/data
06:19:52.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:19:52.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:19:52.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:19:52.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:19:52.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:19:52.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:19:52.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-2/logs
06:19:52.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-2/data/info.pb
06:19:52.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:19:52.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:19:52.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:19:52.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:19:52.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:19:52.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:19:52.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:19:52.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:19:52.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
06:19:52.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:19:52.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:19:52.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:19:52.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.224.124:35387
06:19:52.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.224.124
06:19:52.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:19:52.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/krb5kdc/kudu_127.2.224.124.keytab
06:19:52.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.224.124
06:19:52.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
06:19:52.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
06:19:52.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
06:19:52.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.224.84:37841
06:19:52.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:19:52.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:19:52.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:19:52.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:19:52.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.224.126:38429,127.2.224.125:37335,127.2.224.124:35387
06:19:52.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:19:52.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43029/jwks.json
06:19:52.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
06:19:52.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:52.847769 3121 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.224.125" port: 37335 } has no permanent_uuid. Determining permanent_uuid...
06:19:52.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 06:19:52 dist-test-slave-0tdn krb5kdc[2957](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790489992, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.224.125@KRBTEST.COM for kudu/127.2.224.125@KRBTEST.COM
06:19:52.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:52.877092 3121 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.224.124" port: 35387 } has no permanent_uuid. Determining permanent_uuid...
06:19:52.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:52.879894 3070 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.224.124:35387: connect: Connection refused (error 111)
06:19:52.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:52.883306 3121 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.224.124:35387: Network error: Client connection negotiation failed: client connection to 127.2.224.124:35387: connect: Connection refused (error 111)
06:19:52.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:52.932754 3121 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.224.124" port: 35387 } attempt: 1
06:19:52.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:52.938022 3121 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.224.124:35387: Network error: Client connection negotiation failed: client connection to 127.2.224.124:35387: connect: Connection refused (error 111)
06:19:53.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:53.006421 3121 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.224.124" port: 35387 } attempt: 2
06:19:53.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:53.010255 3121 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.224.124:35387: Network error: Client connection negotiation failed: client connection to 127.2.224.124:35387: connect: Connection refused (error 111)
06:19:53.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:53.101624 3121 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.224.124" port: 35387 } attempt: 3
06:19:53.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:53.105193 3121 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.224.124:35387: Network error: Client connection negotiation failed: client connection to 127.2.224.124:35387: connect: Connection refused (error 111)
06:19:53.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:53.164103 3127 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:19:53.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:53.164608 3127 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:19:53.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:53.165140 3127 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:19:53.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:53.195776 3127 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
06:19:53.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:53.196090 3127 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
06:19:53.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:53.196535 3127 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:19:53.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:53.196897 3127 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
06:19:53.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:53.197221 3127 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
06:19:53.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:53.197533 3127 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
06:19:53.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:53.197674 3127 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43029/jwks.json
06:19:53.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:53.241735 3127 master_runner.cc:386] Master server non-default flags:
06:19:53.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:19:53.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.224.84:37841
06:19:53.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:19:53.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:19:53.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-2/data
06:19:53.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-2/wal
06:19:53.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
06:19:53.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:19:53.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.224.126:38429,127.2.224.125:37335,127.2.224.124:35387
06:19:53.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/krb5kdc/kudu_127.2.224.124.keytab
06:19:53.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.224.124
06:19:53.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:19:53.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:19:53.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:19:53.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.224.124:35387
06:19:53.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:19:53.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:19:53.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:19:53.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43029/jwks.json
06:19:53.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
06:19:53.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
06:19:53.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
06:19:53.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:19:53.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:19:53.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-2/data/info.pb
06:19:53.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.224.124
06:19:53.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:19:53.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:19:53.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:19:53.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:19:53.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:19:53.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:19:53.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-2/logs
06:19:53.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:19:53.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:19:53.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:19:53.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
06:19:53.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:19:53.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:19:53.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:19:53.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 05:19:33 UTC on ccycloud.quasar-ebyiam.root.comops.site
06:19:53.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4675663
06:19:53.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:19:53.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:53.242838 3127 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:19:53.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:53.243825 3127 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:19:53.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:53.248624 3121 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.224.124" port: 35387 } attempt: 4
06:19:53.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:53.253334 3121 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.224.124:35387: Network error: Client connection negotiation failed: client connection to 127.2.224.124:35387: connect: Connection refused (error 111)
06:19:53.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:53.262429 3135 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:19:53.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:53.315414 3042 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.224.125" port: 37335 } attempt: 7
06:19:53.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 06:19:53 dist-test-slave-0tdn krb5kdc[2957](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790489990, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.224.126@KRBTEST.COM for kudu/127.2.224.125@KRBTEST.COM
06:19:53.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:53.359829 3042 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.224.124" port: 35387 } has no permanent_uuid. Determining permanent_uuid...
06:19:53.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:53.364850 3042 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.224.124:35387: Network error: Client connection negotiation failed: client connection to 127.2.224.124:35387: connect: Connection refused (error 111)
06:19:53.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:53.423728 3042 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.224.124" port: 35387 } attempt: 1
06:19:53.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:53.427920 3042 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.224.124:35387: Network error: Client connection negotiation failed: client connection to 127.2.224.124:35387: connect: Connection refused (error 111)
06:19:53.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:53.509302 3042 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.224.124" port: 35387 } attempt: 2
06:19:53.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:53.512849 3042 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.224.124:35387: Network error: Client connection negotiation failed: client connection to 127.2.224.124:35387: connect: Connection refused (error 111)
06:19:53.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:53.552771 3121 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.224.124" port: 35387 } attempt: 5
06:19:53.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:53.557121 3121 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.224.124:35387: Network error: Client connection negotiation failed: client connection to 127.2.224.124:35387: connect: Connection refused (error 111)
06:19:53.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:53.598426 3042 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.224.124" port: 35387 } attempt: 3
06:19:53.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:53.604344 3042 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.224.124:35387: Network error: Client connection negotiation failed: client connection to 127.2.224.124:35387: connect: Connection refused (error 111)
06:19:53.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:53.744895 3042 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.224.124" port: 35387 } attempt: 4
06:19:53.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:53.748739 3042 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.224.124:35387: Network error: Client connection negotiation failed: client connection to 127.2.224.124:35387: connect: Connection refused (error 111)
06:19:54.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:54.032307 3042 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.224.124" port: 35387 } attempt: 5
06:19:54.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:54.036784 3042 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.224.124:35387: Network error: Client connection negotiation failed: client connection to 127.2.224.124:35387: connect: Connection refused (error 111)
06:19:54.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:54.104539 3121 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.224.124" port: 35387 } attempt: 6
06:19:54.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:54.110817 3121 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.224.124:35387: Network error: Client connection negotiation failed: client connection to 127.2.224.124:35387: connect: Connection refused (error 111)
06:19:54.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:53.262632 3138 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:19:54.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:53.262471 3136 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:19:54.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:53.268047 3127 server_base.cc:1034] running on GCE node
06:19:54.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:54.486138 3127 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:19:54.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:54.488792 3127 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:19:54.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:54.490119 3127 hybrid_clock.cc:648] HybridClock initialized: now 1790489994490091 us; error 48 us; skew 500 ppm
06:19:54.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:54.490880 3127 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:19:54.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 06:19:54 dist-test-slave-0tdn krb5kdc[2957](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790489994, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.224.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
06:19:54.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:54.496590 3127 init.cc:377] Logged in from keytab as kudu/127.2.224.124@KRBTEST.COM (short username kudu)
06:19:54.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:54.501361 3127 webserver.cc:460] Webserver started at http://127.2.224.124:45833/ using document root <none> and password file <none>
06:19:54.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:54.502280 3127 fs_manager.cc:359] Metadata directory not provided
06:19:54.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:54.502504 3127 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:19:54.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:54.503006 3127 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:19:54.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:54.507902 3127 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-2/data/instance:
06:19:54.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8c99d7a808bd4466be4b84bb9e1cdadd"
06:19:54.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 06:19:54 on dist-test-slave-0tdn"
06:19:54.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:54.509085 3127 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-2/wal/instance:
06:19:54.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8c99d7a808bd4466be4b84bb9e1cdadd"
06:19:54.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 06:19:54 on dist-test-slave-0tdn"
06:19:54.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:54.516918 3127 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
06:19:54.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:54.522025 3152 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:19:54.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:54.523021 3127 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
06:19:54.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:54.523283 3127 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-2/data,/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-2/wal
06:19:54.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8c99d7a808bd4466be4b84bb9e1cdadd"
06:19:54.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 06:19:54 on dist-test-slave-0tdn"
06:19:54.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:54.523579 3127 fs_report.cc:383] FS layout report
06:19:54.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:19:54.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-2/wal
06:19:54.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-2/wal
06:19:54.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-2/data/data
06:19:54.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:19:54.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:19:54.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:19:54.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:19:54.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:19:54.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:19:54.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:19:54.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:19:54.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:19:54.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:19:54.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:19:54.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:54.556895 3127 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
06:19:54.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:54.562222 3127 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:19:54.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:54.562575 3127 kserver.cc:163] Server-wide thread pool size limit: 3276
06:19:54.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:54.589196 3042 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.224.124" port: 35387 } attempt: 6
06:19:54.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:54.596524 3042 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.224.124:35387: Network error: Client connection negotiation failed: client connection to 127.2.224.124:35387: connect: Connection refused (error 111)
06:19:54.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:54.644829 3127 rpc_server.cc:307] RPC server started. Bound to: 127.2.224.124:35387
06:19:54.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:54.644829 3205 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.2.224.124:35387 every 8 connection(s)
06:19:54.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:54.647601 3127 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-2/data/info.pb
06:19:54.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:54.650497 2945 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskwUNst7/build/tsan/bin/kudu as pid 3127
06:19:54.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:54.650954 2945 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-2/wal/instance
06:19:54.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:54.656287 3206 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
06:19:54.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:54.672731 3206 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.224.126" port: 38429 } has no permanent_uuid. Determining permanent_uuid...
06:19:54.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 06:19:54 dist-test-slave-0tdn krb5kdc[2957](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790489994, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.224.124@KRBTEST.COM for kudu/127.2.224.126@KRBTEST.COM
06:19:54.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 06:19:54 dist-test-slave-0tdn krb5kdc[2957](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790489989, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.2.224.124@KRBTEST.COM
06:19:54.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:54.708887 3206 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.224.125" port: 37335 } has no permanent_uuid. Determining permanent_uuid...
06:19:54.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 06:19:54 dist-test-slave-0tdn krb5kdc[2957](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790489994, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.224.124@KRBTEST.COM for kudu/127.2.224.125@KRBTEST.COM
06:19:54.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 06:19:54 dist-test-slave-0tdn krb5kdc[2957](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790489989, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.2.224.125@KRBTEST.COM
06:19:54.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:54.732815 3206 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.224.124" port: 35387 } has no permanent_uuid. Determining permanent_uuid...
06:19:54.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 06:19:54 dist-test-slave-0tdn krb5kdc[2957](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790489994, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.224.124@KRBTEST.COM for kudu/127.2.224.124@KRBTEST.COM
06:19:54.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 06:19:54 dist-test-slave-0tdn krb5kdc[2957](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790489989, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.2.224.126@KRBTEST.COM
06:19:54.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:54.765628 3206 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd: Bootstrap starting.
06:19:54.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:54.770411 2945 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
06:19:54.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:54.772878 3206 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd: Neither blocks nor log segments found. Creating new log.
06:19:54.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:54.775020 3206 log.cc:824] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd: Log is configured to *not* fsync() on all Append() calls
06:19:54.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:54.780325 3206 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd: No bootstrap required, opened a new log
06:19:54.780 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
06:19:54.782 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
06:19:54.797 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
06:19:54.798 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
06:19:54.801 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.2.224.126"
port: 38429
}
bound_http_address {
host: "127.2.224.126"
port: 45581
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.2.224.125"
port: 37335
}
bound_http_address {
host: "127.2.224.125"
port: 34395
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.2.224.124"
port: 35387
}
bound_http_address {
host: "127.2.224.124"
port: 45833
}
}
}
06:19:54.801 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
06:19:54.802 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
06:19:54.802 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
06:19:54.804 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
06:19:54.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:54.808599 3206 raft_consensus.cc:348] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8726f6ab2f024c2ea9d0e2c1ae24b2cd" member_type: VOTER last_known_addr { host: "127.2.224.126" port: 38429 } } peers { permanent_uuid: "8a628d3d004b41b8beea7b87db236525" member_type: VOTER last_known_addr { host: "127.2.224.125" port: 37335 } } peers { permanent_uuid: "8c99d7a808bd4466be4b84bb9e1cdadd" member_type: VOTER last_known_addr { host: "127.2.224.124" port: 35387 } }
06:19:54.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:54.809634 3206 raft_consensus.cc:374] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:19:54.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:54.810326 3206 raft_consensus.cc:729] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8c99d7a808bd4466be4b84bb9e1cdadd, State: Initialized, Role: FOLLOWER
06:19:54.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:54.811395 3206 consensus_queue.cc:260] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8726f6ab2f024c2ea9d0e2c1ae24b2cd" member_type: VOTER last_known_addr { host: "127.2.224.126" port: 38429 } } peers { permanent_uuid: "8a628d3d004b41b8beea7b87db236525" member_type: VOTER last_known_addr { host: "127.2.224.125" port: 37335 } } peers { permanent_uuid: "8c99d7a808bd4466be4b84bb9e1cdadd" member_type: VOTER last_known_addr { host: "127.2.224.124" port: 35387 } }
06:19:54.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:54.821051 3217 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8726f6ab2f024c2ea9d0e2c1ae24b2cd" member_type: VOTER last_known_addr { host: "127.2.224.126" port: 38429 } } peers { permanent_uuid: "8a628d3d004b41b8beea7b87db236525" member_type: VOTER last_known_addr { host: "127.2.224.125" port: 37335 } } peers { permanent_uuid: "8c99d7a808bd4466be4b84bb9e1cdadd" member_type: VOTER last_known_addr { host: "127.2.224.124" port: 35387 } } }
06:19:54.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:54.821838 3217 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd [sys.catalog]: This master's current role is: FOLLOWER
06:19:54.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:54.822994 3206 sys_catalog.cc:564] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd [sys.catalog]: configured and running, proceeding with master startup.
06:19:54.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 06:19:54 dist-test-slave-0tdn krb5kdc[2957](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790489994, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
06:19:54.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
06:19:54.852 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
06:19:54.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:54.852843 3229 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:19:54.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:54.853163 3229 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:19:54.855 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/krb5kdc/krb5cc
06:19:54.862 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
06:19:54.863 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/krb5kdc/krb5cc
06:19:54.865 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
06:19:54.869 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
06:19:54.871 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
06:19:54.871 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
06:19:54.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:54.848966 3127 master_runner.cc:186] Error getting master registration for 127.2.224.126:38429: OK, instance_id { permanent_uuid: "8726f6ab2f024c2ea9d0e2c1ae24b2cd" instance_seqno: 1790489990926325 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
06:19:54.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:54.880225 3127 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:19:54.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 06:19:54 dist-test-slave-0tdn krb5kdc[2957](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790489994, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
06:19:54.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
06:19:54.902 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
06:19:54.903 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/krb5kdc/krb5cc
06:19:54.904 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
06:19:54.905 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.2.224.124:35387
06:19:54.907 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
06:19:54.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:54.907933 2945 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskwUNst7/build/tsan/bin/kudu with pid 3127
06:19:54.929 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
06:19:54.929 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.2.224.125:37335
06:19:54.929 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
06:19:54.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:54.930032 2945 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskwUNst7/build/tsan/bin/kudu with pid 3048
06:19:54.953 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
06:19:54.953 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.2.224.126:38429
06:19:54.953 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
06:19:54.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:54.953843 2945 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskwUNst7/build/tsan/bin/kudu with pid 2971
06:19:54.976 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
06:19:54.976 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.2.224.124:35387
06:19:54.977 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
06:19:54.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:54.978991 2945 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskwUNst7/build/tsan/bin/kudu
06:19:54.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskwUNst7/build/tsan/bin/kudu
06:19:54.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-2/wal
06:19:54.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-2/data
06:19:54.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:19:54.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:19:54.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:19:54.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:19:54.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:19:54.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:19:54.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-2/logs
06:19:54.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-2/data/info.pb
06:19:54.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:19:54.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:19:54.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:19:54.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:19:54.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:19:54.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:19:54.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:19:54.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:19:54.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
06:19:54.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:19:54.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:19:54.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:19:54.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.224.124:35387
06:19:54.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.224.124
06:19:54.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45833
06:19:54.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/krb5kdc/kudu_127.2.224.124.keytab
06:19:54.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.224.124
06:19:54.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
06:19:54.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
06:19:54.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
06:19:54.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.224.84:37841
06:19:54.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:19:54.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:19:54.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:19:54.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:19:54.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.224.126:38429,127.2.224.125:37335,127.2.224.124:35387
06:19:54.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:19:54.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43029/jwks.json
06:19:54.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
06:19:55.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:55.292142 3233 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:19:55.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:55.292660 3233 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:19:55.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:55.293206 3233 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:19:55.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:55.322198 3233 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
06:19:55.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:55.322409 3233 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
06:19:55.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:55.322664 3233 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:19:55.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:55.322849 3233 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
06:19:55.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:55.323040 3233 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
06:19:55.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:55.323207 3233 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
06:19:55.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:55.323330 3233 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43029/jwks.json
06:19:55.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:55.357607 3233 master_runner.cc:386] Master server non-default flags:
06:19:55.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:19:55.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.224.84:37841
06:19:55.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:19:55.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:19:55.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-2/data
06:19:55.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-2/wal
06:19:55.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
06:19:55.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:19:55.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.224.126:38429,127.2.224.125:37335,127.2.224.124:35387
06:19:55.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/krb5kdc/kudu_127.2.224.124.keytab
06:19:55.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.224.124
06:19:55.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:19:55.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:19:55.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:19:55.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.224.124:35387
06:19:55.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:19:55.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:19:55.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:19:55.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43029/jwks.json
06:19:55.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
06:19:55.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
06:19:55.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
06:19:55.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:19:55.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:19:55.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-2/data/info.pb
06:19:55.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.224.124
06:19:55.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45833
06:19:55.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:19:55.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:19:55.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:19:55.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:19:55.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:19:55.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-2/logs
06:19:55.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:19:55.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:19:55.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:19:55.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
06:19:55.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:19:55.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:19:55.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:19:55.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 05:19:33 UTC on ccycloud.quasar-ebyiam.root.comops.site
06:19:55.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4675663
06:19:55.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:19:55.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:55.358850 3233 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:19:55.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:55.359719 3233 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:19:55.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:55.376583 3239 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:19:56.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:55.376904 3242 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:19:56.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:55.378463 3233 server_base.cc:1034] running on GCE node
06:19:56.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:55.377823 3240 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:19:56.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:56.595880 3233 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:19:56.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:56.598302 3233 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:19:56.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:56.599663 3233 hybrid_clock.cc:648] HybridClock initialized: now 1790489996599632 us; error 59 us; skew 500 ppm
06:19:56.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:56.600494 3233 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:19:56.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 06:19:56 dist-test-slave-0tdn krb5kdc[2957](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790489996, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.224.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
06:19:56.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:56.606110 3233 init.cc:377] Logged in from keytab as kudu/127.2.224.124@KRBTEST.COM (short username kudu)
06:19:56.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:56.611200 3233 webserver.cc:460] Webserver started at http://127.2.224.124:45833/ using document root <none> and password file <none>
06:19:56.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:56.612155 3233 fs_manager.cc:359] Metadata directory not provided
06:19:56.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:56.612401 3233 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:19:56.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:56.620014 3233 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.006s sys 0.001s
06:19:56.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:56.624145 3252 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:19:56.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:56.625108 3233 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
06:19:56.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:56.625437 3233 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-2/data,/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-2/wal
06:19:56.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8c99d7a808bd4466be4b84bb9e1cdadd"
06:19:56.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 06:19:54 on dist-test-slave-0tdn"
06:19:56.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:56.627614 3233 fs_report.cc:383] FS layout report
06:19:56.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:19:56.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-2/wal
06:19:56.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-2/wal
06:19:56.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-2/data/data
06:19:56.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:19:56.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:19:56.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:19:56.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:19:56.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:19:56.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:19:56.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:19:56.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:19:56.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:19:56.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:19:56.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:19:56.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:56.668272 3233 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
06:19:56.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:56.674019 3233 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:19:56.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:56.674587 3233 kserver.cc:163] Server-wide thread pool size limit: 3276
06:19:56.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:56.742020 3233 rpc_server.cc:307] RPC server started. Bound to: 127.2.224.124:35387
06:19:56.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:56.742151 3303 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.2.224.124:35387 every 8 connection(s)
06:19:56.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:56.744799 3233 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-2/data/info.pb
06:19:56.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:56.749444 2945 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskwUNst7/build/tsan/bin/kudu as pid 3233
06:19:56.750 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
06:19:56.750 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.2.224.125:37335
06:19:56.750 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
06:19:56.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:56.751699 2945 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskwUNst7/build/tsan/bin/kudu
06:19:56.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskwUNst7/build/tsan/bin/kudu
06:19:56.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-1/wal
06:19:56.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-1/data
06:19:56.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:19:56.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:19:56.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:19:56.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:19:56.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:19:56.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:19:56.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-1/logs
06:19:56.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-1/data/info.pb
06:19:56.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:19:56.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:19:56.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:19:56.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:19:56.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:19:56.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:19:56.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:19:56.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:19:56.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
06:19:56.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:19:56.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:19:56.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:19:56.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.224.125:37335
06:19:56.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.224.125
06:19:56.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34395
06:19:56.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/krb5kdc/kudu_127.2.224.125.keytab
06:19:56.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.224.125
06:19:56.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
06:19:56.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
06:19:56.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
06:19:56.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.224.84:37841
06:19:56.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:19:56.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:19:56.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:19:56.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:19:56.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.224.126:38429,127.2.224.125:37335,127.2.224.124:35387
06:19:56.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:19:56.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43029/jwks.json
06:19:56.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
06:19:56.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:56.755470 3304 sys_catalog.cc:263] Verifying existing consensus state
06:19:56.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:56.761066 3304 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd: Bootstrap starting.
06:19:56.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:56.792887 3304 log.cc:824] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd: Log is configured to *not* fsync() on all Append() calls
06:19:56.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:56.797194 3304 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
06:19:56.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:56.797786 3304 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd: Bootstrap complete.
06:19:56.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:56.820004 3304 raft_consensus.cc:348] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8726f6ab2f024c2ea9d0e2c1ae24b2cd" member_type: VOTER last_known_addr { host: "127.2.224.126" port: 38429 } } peers { permanent_uuid: "8a628d3d004b41b8beea7b87db236525" member_type: VOTER last_known_addr { host: "127.2.224.125" port: 37335 } } peers { permanent_uuid: "8c99d7a808bd4466be4b84bb9e1cdadd" member_type: VOTER last_known_addr { host: "127.2.224.124" port: 35387 } }
06:19:56.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:56.822022 3304 raft_consensus.cc:374] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:19:56.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:56.822274 3304 raft_consensus.cc:729] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8c99d7a808bd4466be4b84bb9e1cdadd, State: Initialized, Role: FOLLOWER
06:19:56.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:56.822985 3304 consensus_queue.cc:260] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8726f6ab2f024c2ea9d0e2c1ae24b2cd" member_type: VOTER last_known_addr { host: "127.2.224.126" port: 38429 } } peers { permanent_uuid: "8a628d3d004b41b8beea7b87db236525" member_type: VOTER last_known_addr { host: "127.2.224.125" port: 37335 } } peers { permanent_uuid: "8c99d7a808bd4466be4b84bb9e1cdadd" member_type: VOTER last_known_addr { host: "127.2.224.124" port: 35387 } }
06:19:56.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:56.825464 3308 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8726f6ab2f024c2ea9d0e2c1ae24b2cd" member_type: VOTER last_known_addr { host: "127.2.224.126" port: 38429 } } peers { permanent_uuid: "8a628d3d004b41b8beea7b87db236525" member_type: VOTER last_known_addr { host: "127.2.224.125" port: 37335 } } peers { permanent_uuid: "8c99d7a808bd4466be4b84bb9e1cdadd" member_type: VOTER last_known_addr { host: "127.2.224.124" port: 35387 } } }
06:19:56.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:56.826587 3304 sys_catalog.cc:564] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd [sys.catalog]: configured and running, proceeding with master startup.
06:19:56.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:56.826520 3308 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd [sys.catalog]: This master's current role is: FOLLOWER
06:19:56.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:56.856750 3319 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:19:56.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:56.857462 3319 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:19:56.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:56.866694 3308 raft_consensus.cc:482] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:19:56.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:56.867216 3308 raft_consensus.cc:504] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8726f6ab2f024c2ea9d0e2c1ae24b2cd" member_type: VOTER last_known_addr { host: "127.2.224.126" port: 38429 } } peers { permanent_uuid: "8a628d3d004b41b8beea7b87db236525" member_type: VOTER last_known_addr { host: "127.2.224.125" port: 37335 } } peers { permanent_uuid: "8c99d7a808bd4466be4b84bb9e1cdadd" member_type: VOTER last_known_addr { host: "127.2.224.124" port: 35387 } }
06:19:56.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:56.871369 3308 leader_election.cc:288] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8726f6ab2f024c2ea9d0e2c1ae24b2cd (127.2.224.126:38429), 8a628d3d004b41b8beea7b87db236525 (127.2.224.125:37335)
06:19:56.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:56.873693 3256 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.224.125:37335: connect: Connection refused (error 111)
06:19:56.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:56.882936 3256 leader_election.cc:334] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 8a628d3d004b41b8beea7b87db236525 (127.2.224.125:37335): Network error: Client connection negotiation failed: client connection to 127.2.224.125:37335: connect: Connection refused (error 111)
06:19:56.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:56.895078 3254 leader_election.cc:334] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 8726f6ab2f024c2ea9d0e2c1ae24b2cd (127.2.224.126:38429): Network error: Client connection negotiation failed: client connection to 127.2.224.126:38429: connect: Connection refused (error 111)
06:19:56.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:56.895568 3254 leader_election.cc:302] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 8c99d7a808bd4466be4b84bb9e1cdadd; no voters: 8726f6ab2f024c2ea9d0e2c1ae24b2cd, 8a628d3d004b41b8beea7b87db236525
06:19:56.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:56.896751 3308 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
06:19:56.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:56.859670 3233 master_runner.cc:186] Error getting master registration for 127.2.224.126:38429: Network error: Client connection negotiation failed: client connection to 127.2.224.126:38429: connect: Connection refused (error 111),
06:19:56.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:56.904482 3233 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:19:56.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:56.929719 3258 tablet.cc:2369] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd: Can't schedule compaction. Clean time has not been advanced past its initial value.
06:19:57.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:57.089165 3306 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:19:57.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:57.089735 3306 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:19:57.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:57.090366 3306 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:19:57.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:57.122862 3306 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
06:19:57.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:57.123173 3306 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
06:19:57.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:57.123582 3306 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:19:57.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:57.123930 3306 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
06:19:57.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:57.124244 3306 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
06:19:57.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:57.124548 3306 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
06:19:57.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:57.124785 3306 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43029/jwks.json
06:19:57.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:57.170212 3306 master_runner.cc:386] Master server non-default flags:
06:19:57.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:19:57.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.224.84:37841
06:19:57.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:19:57.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:19:57.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-1/data
06:19:57.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-1/wal
06:19:57.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
06:19:57.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:19:57.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.224.126:38429,127.2.224.125:37335,127.2.224.124:35387
06:19:57.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/krb5kdc/kudu_127.2.224.125.keytab
06:19:57.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.224.125
06:19:57.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:19:57.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:19:57.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:19:57.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.224.125:37335
06:19:57.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:19:57.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:19:57.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:19:57.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43029/jwks.json
06:19:57.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
06:19:57.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
06:19:57.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
06:19:57.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:19:57.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:19:57.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-1/data/info.pb
06:19:57.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.224.125
06:19:57.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34395
06:19:57.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:19:57.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:19:57.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:19:57.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:19:57.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:19:57.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-1/logs
06:19:57.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:19:57.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:19:57.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:19:57.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
06:19:57.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:19:57.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:19:57.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:19:57.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 05:19:33 UTC on ccycloud.quasar-ebyiam.root.comops.site
06:19:57.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4675663
06:19:57.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:19:57.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:57.171754 3306 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:19:57.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:57.173138 3306 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:19:57.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:57.193128 3330 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:19:57.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:57.861882 3319 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:19:57.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:57.862249 3319 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:19:58.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:57.194382 3329 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:19:58.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:57.204511 3332 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:19:58.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:58.772964 3331 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1580 milliseconds
06:19:58.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:58.773066 3306 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
06:19:58.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:58.778759 3306 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:19:58.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:58.781950 3306 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:19:58.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:58.783368 3306 hybrid_clock.cc:648] HybridClock initialized: now 1790489998783321 us; error 62 us; skew 500 ppm
06:19:58.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:58.784677 3306 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:19:58.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 06:19:58 dist-test-slave-0tdn krb5kdc[2957](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790489998, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.224.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
06:19:58.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:58.795593 3306 init.cc:377] Logged in from keytab as kudu/127.2.224.125@KRBTEST.COM (short username kudu)
06:19:58.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:58.801393 3306 webserver.cc:460] Webserver started at http://127.2.224.125:34395/ using document root <none> and password file <none>
06:19:58.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:58.802240 3306 fs_manager.cc:359] Metadata directory not provided
06:19:58.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:58.802436 3306 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:19:58.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:58.810647 3306 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.005s sys 0.000s
06:19:58.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:58.815243 3343 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:19:58.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:58.816334 3306 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
06:19:58.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:58.816601 3306 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-1/data,/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-1/wal
06:19:58.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8a628d3d004b41b8beea7b87db236525"
06:19:58.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 06:19:52 on dist-test-slave-0tdn"
06:19:58.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:58.819007 3306 fs_report.cc:383] FS layout report
06:19:58.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:19:58.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-1/wal
06:19:58.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-1/wal
06:19:58.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-1/data/data
06:19:58.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:19:58.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:19:58.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:19:58.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:19:58.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:19:58.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:19:58.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:19:58.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:19:58.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:19:58.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:19:58.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:19:58.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:58.821024 3344 raft_consensus.cc:482] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:19:58.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:58.821484 3344 raft_consensus.cc:504] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8726f6ab2f024c2ea9d0e2c1ae24b2cd" member_type: VOTER last_known_addr { host: "127.2.224.126" port: 38429 } } peers { permanent_uuid: "8a628d3d004b41b8beea7b87db236525" member_type: VOTER last_known_addr { host: "127.2.224.125" port: 37335 } } peers { permanent_uuid: "8c99d7a808bd4466be4b84bb9e1cdadd" member_type: VOTER last_known_addr { host: "127.2.224.124" port: 35387 } }
06:19:58.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:58.823559 3344 leader_election.cc:288] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8726f6ab2f024c2ea9d0e2c1ae24b2cd (127.2.224.126:38429), 8a628d3d004b41b8beea7b87db236525 (127.2.224.125:37335)
06:19:58.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:58.832576 3254 leader_election.cc:334] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 8726f6ab2f024c2ea9d0e2c1ae24b2cd (127.2.224.126:38429): Network error: Client connection negotiation failed: client connection to 127.2.224.126:38429: connect: Connection refused (error 111)
06:19:58.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:58.833287 3256 leader_election.cc:334] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 8a628d3d004b41b8beea7b87db236525 (127.2.224.125:37335): Network error: Client connection negotiation failed: client connection to 127.2.224.125:37335: connect: Connection refused (error 111)
06:19:58.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:58.833724 3256 leader_election.cc:302] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 8c99d7a808bd4466be4b84bb9e1cdadd; no voters: 8726f6ab2f024c2ea9d0e2c1ae24b2cd, 8a628d3d004b41b8beea7b87db236525
06:19:58.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:58.834426 3344 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
06:19:58.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:58.866245 3319 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:19:58.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:58.866544 3319 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:19:58.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:58.872356 3306 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
06:19:58.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:58.878567 3306 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:19:58.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:58.878954 3306 kserver.cc:163] Server-wide thread pool size limit: 3276
06:19:58.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:58.969069 3306 rpc_server.cc:307] RPC server started. Bound to: 127.2.224.125:37335
06:19:58.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:58.969170 3398 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.2.224.125:37335 every 8 connection(s)
06:19:58.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:58.972723 3306 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-1/data/info.pb
06:19:58.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:58.978384 2945 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskwUNst7/build/tsan/bin/kudu as pid 3306
06:19:58.980 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
06:19:58.980 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.2.224.126:38429
06:19:58.981 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
06:19:58.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:58.984004 2945 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskwUNst7/build/tsan/bin/kudu
06:19:58.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskwUNst7/build/tsan/bin/kudu
06:19:58.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-0/wal
06:19:58.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-0/data
06:19:58.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:19:58.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:19:58.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:19:58.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:19:58.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:19:58.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:19:58.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-0/logs
06:19:58.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-0/data/info.pb
06:19:58.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:19:58.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:19:58.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:19:58.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:19:58.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:19:58.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:19:58.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:19:58.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:19:58.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
06:19:58.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:19:58.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:19:58.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:19:58.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.224.126:38429
06:19:58.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.224.126
06:19:58.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45581
06:19:58.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/krb5kdc/kudu_127.2.224.126.keytab
06:19:58.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.224.126
06:19:58.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
06:19:58.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
06:19:58.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
06:19:58.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.224.84:37841
06:19:58.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:19:58.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:19:58.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:19:58.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:19:58.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.224.126:38429,127.2.224.125:37335,127.2.224.124:35387
06:19:58.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:19:58.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43029/jwks.json
06:19:58.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
06:19:58.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:58.990381 3399 sys_catalog.cc:263] Verifying existing consensus state
06:19:58.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260927 06:19:58.991319 3399 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
06:19:58.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:58.991743 3306 master.cc:537] Master@127.2.224.125:37335 shutting down...
06:19:59.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:59.063642 3306 master.cc:552] Master@127.2.224.125:37335 shutdown complete.
06:19:59.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
06:19:59.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:59.336066 3401 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:19:59.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:59.336824 3401 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:19:59.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:59.337543 3401 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:19:59.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:59.369307 3401 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
06:19:59.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:59.369585 3401 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
06:19:59.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:59.369951 3401 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:19:59.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:59.370260 3401 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
06:19:59.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:59.370575 3401 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
06:19:59.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:59.370879 3401 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
06:19:59.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:59.371204 3401 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43029/jwks.json
06:19:59.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:59.410262 3401 master_runner.cc:386] Master server non-default flags:
06:19:59.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:19:59.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.224.84:37841
06:19:59.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:19:59.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:19:59.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-0/data
06:19:59.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-0/wal
06:19:59.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
06:19:59.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:19:59.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.224.126:38429,127.2.224.125:37335,127.2.224.124:35387
06:19:59.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/krb5kdc/kudu_127.2.224.126.keytab
06:19:59.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.224.126
06:19:59.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:19:59.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:19:59.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:19:59.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.224.126:38429
06:19:59.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:19:59.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:19:59.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:19:59.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43029/jwks.json
06:19:59.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
06:19:59.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
06:19:59.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
06:19:59.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:19:59.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:19:59.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-0/data/info.pb
06:19:59.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.224.126
06:19:59.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45581
06:19:59.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:19:59.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:19:59.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:19:59.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:19:59.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:19:59.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-0/logs
06:19:59.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:19:59.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:19:59.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:19:59.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
06:19:59.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:19:59.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:19:59.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:19:59.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 05:19:33 UTC on ccycloud.quasar-ebyiam.root.comops.site
06:19:59.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4675663
06:19:59.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:19:59.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:59.411458 3401 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:19:59.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:59.412817 3401 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:19:59.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:59.439420 3408 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:19:59.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:59.869417 3319 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:19:59.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:59.869680 3319 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:20:00.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:59.442173 3410 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:20:00.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:19:59.442735 3401 server_base.cc:1034] running on GCE node
06:20:00.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:19:59.440714 3407 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:20:00.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:00.658143 3401 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:20:00.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:00.660707 3401 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:20:00.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:00.662168 3401 hybrid_clock.cc:648] HybridClock initialized: now 1790490000662103 us; error 80 us; skew 500 ppm
06:20:00.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:00.663038 3401 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:20:00.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 06:20:00 dist-test-slave-0tdn krb5kdc[2957](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790490000, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.224.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
06:20:00.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:00.669139 3401 init.cc:377] Logged in from keytab as kudu/127.2.224.126@KRBTEST.COM (short username kudu)
06:20:00.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:00.674924 3401 webserver.cc:460] Webserver started at http://127.2.224.126:45581/ using document root <none> and password file <none>
06:20:00.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:00.675951 3401 fs_manager.cc:359] Metadata directory not provided
06:20:00.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:00.676168 3401 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:20:00.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:00.683553 3401 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.006s sys 0.001s
06:20:00.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:00.687853 3418 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:20:00.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:00.688941 3401 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
06:20:00.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:00.689224 3401 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-0/data,/tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-0/wal
06:20:00.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8726f6ab2f024c2ea9d0e2c1ae24b2cd"
06:20:00.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 06:19:50 on dist-test-slave-0tdn"
06:20:00.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:00.691313 3401 fs_report.cc:383] FS layout report
06:20:00.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:20:00.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-0/wal
06:20:00.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-0/wal
06:20:00.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-0/data/data
06:20:00.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:20:00.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:20:00.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:20:00.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:20:00.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:20:00.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:20:00.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:20:00.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:20:00.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:20:00.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:20:00.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:20:00.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:00.745247 3401 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
06:20:00.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:00.751055 3401 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:20:00.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:00.751500 3401 kserver.cc:163] Server-wide thread pool size limit: 3276
06:20:00.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:00.823045 3401 rpc_server.cc:307] RPC server started. Bound to: 127.2.224.126:38429
06:20:00.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:00.823177 3470 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.2.224.126:38429 every 8 connection(s)
06:20:00.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:00.825821 3401 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-0/data/info.pb
06:20:00.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:00.833978 2945 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskwUNst7/build/tsan/bin/kudu as pid 3401
06:20:00.834 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
06:20:00.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:00.837301 3471 sys_catalog.cc:263] Verifying existing consensus state
06:20:00.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260927 06:20:00.838147 3471 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
06:20:00.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:00.838651 3401 master.cc:537] Master@127.2.224.126:38429 shutting down...
06:20:00.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:00.872718 3319 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:20:00.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:00.872979 3319 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:20:00.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:00.881820 3474 raft_consensus.cc:482] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:20:00.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:00.882464 3474 raft_consensus.cc:504] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8726f6ab2f024c2ea9d0e2c1ae24b2cd" member_type: VOTER last_known_addr { host: "127.2.224.126" port: 38429 } } peers { permanent_uuid: "8a628d3d004b41b8beea7b87db236525" member_type: VOTER last_known_addr { host: "127.2.224.125" port: 37335 } } peers { permanent_uuid: "8c99d7a808bd4466be4b84bb9e1cdadd" member_type: VOTER last_known_addr { host: "127.2.224.124" port: 35387 } }
06:20:00.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:00.884570 3474 leader_election.cc:288] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8726f6ab2f024c2ea9d0e2c1ae24b2cd (127.2.224.126:38429), 8a628d3d004b41b8beea7b87db236525 (127.2.224.125:37335)
06:20:00.905 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.2.224.124:35387(127.2.224.124:35387)] connecting to peer
06:20:00.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:00.911397 3254 leader_election.cc:334] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 8726f6ab2f024c2ea9d0e2c1ae24b2cd (127.2.224.126:38429): Network error: Client connection negotiation failed: client connection to 127.2.224.126:38429: connect: Connection refused (error 111)
06:20:00.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:00.911880 3256 leader_election.cc:334] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 8a628d3d004b41b8beea7b87db236525 (127.2.224.125:37335): Network error: Client connection negotiation failed: client connection to 127.2.224.125:37335: connect: Connection refused (error 111)
06:20:00.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:00.912288 3256 leader_election.cc:302] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 8c99d7a808bd4466be4b84bb9e1cdadd; no voters: 8726f6ab2f024c2ea9d0e2c1ae24b2cd, 8a628d3d004b41b8beea7b87db236525
06:20:00.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:00.913045 3474 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
06:20:00.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:00.931908 3401 master.cc:552] Master@127.2.224.126:38429 shutdown complete.
06:20:00.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskwUNst7/test-tmp/mini-kudu-cluster8151819505641226965/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
06:20:00.980 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.2.224.125:37335(127.2.224.125:37335)] connecting to peer
06:20:00.981 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.2.224.126:38429(127.2.224.126:38429)] connecting to peer
06:20:01.036 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.2.224.124:35387(127.2.224.124:35387)] Successfully connected to peer
06:20:01.039 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:01.040 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.224.125:37335(127.2.224.125:37335)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:01.042 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.224.125:37335: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:01.091 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:01.091 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.224.126:38429(127.2.224.126:38429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:01.091 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.224.126:38429: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:01.108 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
06:20:01.116 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.2.224.124
06:20:01.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 06:20:01 dist-test-slave-0tdn krb5kdc[2957](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790489994, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.2.224.124@KRBTEST.COM
06:20:01.339 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x4ab62d57, L:/127.0.0.1:44380 - R:/127.2.224.124:35387] using SASL/GSSAPI
06:20:01.353 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.2.224.124:35387,127.2.224.125:37335,127.2.224.126:38429; will retry
06:20:01.362 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.224.125:37335(127.2.224.125:37335)] connecting to peer
06:20:01.365 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.224.126:38429(127.2.224.126:38429)] connecting to peer
06:20:01.365 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:01.365 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.224.125:37335(127.2.224.125:37335)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:01.365 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.224.125:37335: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:01.368 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:01.369 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.224.126:38429(127.2.224.126:38429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:01.369 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.224.126:38429: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:01.370 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.2.224.124:35387,127.2.224.125:37335,127.2.224.126:38429; will retry
06:20:01.380 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.224.125:37335(127.2.224.125:37335)] connecting to peer
06:20:01.382 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.224.126:38429(127.2.224.126:38429)] connecting to peer
06:20:01.382 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:01.383 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.224.125:37335(127.2.224.125:37335)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:01.384 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.224.125:37335: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:01.386 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:01.386 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.224.126:38429(127.2.224.126:38429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:01.387 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.224.126:38429: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:01.388 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.2.224.124:35387,127.2.224.125:37335,127.2.224.126:38429; will retry
06:20:01.400 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.224.125:37335(127.2.224.125:37335)] connecting to peer
06:20:01.401 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.224.126:38429(127.2.224.126:38429)] connecting to peer
06:20:01.403 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:01.404 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.224.125:37335(127.2.224.125:37335)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:01.405 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.224.125:37335: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:01.406 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:01.406 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.224.126:38429(127.2.224.126:38429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:01.407 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.224.126:38429: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:01.408 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.2.224.124:35387,127.2.224.125:37335,127.2.224.126:38429; will retry
06:20:01.419 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.224.125:37335(127.2.224.125:37335)] connecting to peer
06:20:01.423 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:01.423 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.224.125:37335(127.2.224.125:37335)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:01.424 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.224.125:37335: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:01.424 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.224.126:38429(127.2.224.126:38429)] connecting to peer
06:20:01.426 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:01.426 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.224.126:38429(127.2.224.126:38429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:01.426 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.224.126:38429: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:01.427 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.2.224.124:35387,127.2.224.125:37335,127.2.224.126:38429; will retry
06:20:01.460 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.224.125:37335(127.2.224.125:37335)] connecting to peer
06:20:01.462 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.224.126:38429(127.2.224.126:38429)] connecting to peer
06:20:01.463 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:01.464 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.224.125:37335(127.2.224.125:37335)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:01.465 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.224.125:37335: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:01.468 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:01.468 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.224.126:38429(127.2.224.126:38429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:01.468 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.224.126:38429: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:01.468 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.2.224.124:35387,127.2.224.125:37335,127.2.224.126:38429; will retry
06:20:01.540 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.224.125:37335(127.2.224.125:37335)] connecting to peer
06:20:01.541 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.224.126:38429(127.2.224.126:38429)] connecting to peer
06:20:01.544 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:01.544 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.224.125:37335(127.2.224.125:37335)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:01.545 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:01.545 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.224.126:38429(127.2.224.126:38429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:01.545 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.224.125:37335: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:01.546 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.224.126:38429: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:01.546 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.2.224.124:35387,127.2.224.125:37335,127.2.224.126:38429; will retry
06:20:01.660 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.224.125:37335(127.2.224.125:37335)] connecting to peer
06:20:01.662 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.224.126:38429(127.2.224.126:38429)] connecting to peer
06:20:01.664 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:01.664 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.224.125:37335(127.2.224.125:37335)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:01.665 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.224.125:37335: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:01.666 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:01.666 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.224.126:38429(127.2.224.126:38429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:01.666 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.224.126:38429: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:01.667 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.2.224.124:35387,127.2.224.125:37335,127.2.224.126:38429; will retry
06:20:01.860 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.224.125:37335(127.2.224.125:37335)] connecting to peer
06:20:01.862 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.224.126:38429(127.2.224.126:38429)] connecting to peer
06:20:01.862 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:01.862 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.224.125:37335(127.2.224.125:37335)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:01.863 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.224.125:37335: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:01.866 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:01.866 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.224.126:38429(127.2.224.126:38429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:01.866 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.224.126:38429: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:01.867 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.2.224.124:35387,127.2.224.125:37335,127.2.224.126:38429; will retry
06:20:01.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:01.875960 3319 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:20:01.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:01.876230 3319 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:20:01.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:01.907193 3233 master_runner.cc:186] Error getting master registration for 127.2.224.126:38429: Network error: Client connection negotiation failed: client connection to 127.2.224.126:38429: connect: Connection refused (error 111),
06:20:01.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:01.907585 3233 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:20:01.979 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.224.125:37335(127.2.224.125:37335)] connecting to peer
06:20:01.981 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:01.982 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.224.125:37335(127.2.224.125:37335)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:01.982 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.224.126:38429(127.2.224.126:38429)] connecting to peer
06:20:01.983 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.224.125:37335: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:01.984 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:01.985 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.224.126:38429(127.2.224.126:38429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:01.985 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.224.126:38429: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:01.986 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.2.224.124:35387,127.2.224.125:37335,127.2.224.126:38429; will retry
06:20:02.100 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.224.125:37335(127.2.224.125:37335)] connecting to peer
06:20:02.101 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.224.126:38429(127.2.224.126:38429)] connecting to peer
06:20:02.102 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:02.102 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.224.125:37335(127.2.224.125:37335)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:02.103 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.224.125:37335: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:02.103 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:02.103 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.224.126:38429(127.2.224.126:38429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:02.103 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.224.126:38429: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:02.104 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.2.224.124:35387,127.2.224.125:37335,127.2.224.126:38429; will retry
06:20:02.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:02.879015 3319 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:20:02.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:02.879297 3319 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:20:02.920 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.224.125:37335(127.2.224.125:37335)] connecting to peer
06:20:02.921 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.224.126:38429(127.2.224.126:38429)] connecting to peer
06:20:02.923 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:02.923 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.224.125:37335(127.2.224.125:37335)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:02.923 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.224.125:37335: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:02.924 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:02.924 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.224.126:38429(127.2.224.126:38429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:02.924 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.224.126:38429: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:02.924 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.2.224.124:35387,127.2.224.125:37335,127.2.224.126:38429; will retry
06:20:03.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:03.882222 3319 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:20:03.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:03.882503 3319 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:20:03.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:03.898959 3483 raft_consensus.cc:482] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:20:03.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:03.899415 3483 raft_consensus.cc:504] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8726f6ab2f024c2ea9d0e2c1ae24b2cd" member_type: VOTER last_known_addr { host: "127.2.224.126" port: 38429 } } peers { permanent_uuid: "8a628d3d004b41b8beea7b87db236525" member_type: VOTER last_known_addr { host: "127.2.224.125" port: 37335 } } peers { permanent_uuid: "8c99d7a808bd4466be4b84bb9e1cdadd" member_type: VOTER last_known_addr { host: "127.2.224.124" port: 35387 } }
06:20:03.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:03.901088 3483 leader_election.cc:288] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8726f6ab2f024c2ea9d0e2c1ae24b2cd (127.2.224.126:38429), 8a628d3d004b41b8beea7b87db236525 (127.2.224.125:37335)
06:20:03.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:03.903079 3254 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.224.126:38429: connect: Connection refused (error 111) [suppressed 6 similar messages]
06:20:03.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:03.905507 3256 leader_election.cc:334] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 8a628d3d004b41b8beea7b87db236525 (127.2.224.125:37335): Network error: Client connection negotiation failed: client connection to 127.2.224.125:37335: connect: Connection refused (error 111)
06:20:03.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:03.905869 3254 leader_election.cc:334] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 8726f6ab2f024c2ea9d0e2c1ae24b2cd (127.2.224.126:38429): Network error: Client connection negotiation failed: client connection to 127.2.224.126:38429: connect: Connection refused (error 111)
06:20:03.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:03.906129 3254 leader_election.cc:302] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 8c99d7a808bd4466be4b84bb9e1cdadd; no voters: 8726f6ab2f024c2ea9d0e2c1ae24b2cd, 8a628d3d004b41b8beea7b87db236525
06:20:03.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:03.906594 3483 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
06:20:04.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:04.885712 3319 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:20:04.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:04.885978 3319 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:20:05.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:05.889432 3319 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:20:05.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:05.889801 3319 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:20:06.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 06:20:06 dist-test-slave-0tdn krb5kdc[2957](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790490006, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.224.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
06:20:06.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:06.609974 3248 init.cc:295] Successfully reacquired a new kerberos TGT
06:20:06.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:06.892652 3319 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:20:06.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:06.893049 3319 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:20:06.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:06.910897 3233 master_runner.cc:186] Error getting master registration for 127.2.224.126:38429: Network error: Client connection negotiation failed: client connection to 127.2.224.126:38429: connect: Connection refused (error 111),
06:20:06.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:06.911281 3233 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:20:06.980 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.224.125:37335(127.2.224.125:37335)] connecting to peer
06:20:06.982 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:06.982 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.224.125:37335(127.2.224.125:37335)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:06.982 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.224.125:37335: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:06.982 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.224.126:38429(127.2.224.126:38429)] connecting to peer
06:20:06.985 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:06.985 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.224.126:38429(127.2.224.126:38429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:06.985 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.224.126:38429: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:06.985 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.2.224.124:35387,127.2.224.125:37335,127.2.224.126:38429; will retry
06:20:07.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:07.895776 3319 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:20:07.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:07.896082 3319 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:20:08.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:08.898809 3319 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:20:08.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:08.899129 3319 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:20:08.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:08.929046 3496 raft_consensus.cc:482] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:20:08.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:08.929551 3496 raft_consensus.cc:504] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8726f6ab2f024c2ea9d0e2c1ae24b2cd" member_type: VOTER last_known_addr { host: "127.2.224.126" port: 38429 } } peers { permanent_uuid: "8a628d3d004b41b8beea7b87db236525" member_type: VOTER last_known_addr { host: "127.2.224.125" port: 37335 } } peers { permanent_uuid: "8c99d7a808bd4466be4b84bb9e1cdadd" member_type: VOTER last_known_addr { host: "127.2.224.124" port: 35387 } }
06:20:08.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:08.931372 3496 leader_election.cc:288] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8726f6ab2f024c2ea9d0e2c1ae24b2cd (127.2.224.126:38429), 8a628d3d004b41b8beea7b87db236525 (127.2.224.125:37335)
06:20:08.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:08.933610 3254 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.224.126:38429: connect: Connection refused (error 111)
06:20:08.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:08.936769 3254 leader_election.cc:334] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 8726f6ab2f024c2ea9d0e2c1ae24b2cd (127.2.224.126:38429): Network error: Client connection negotiation failed: client connection to 127.2.224.126:38429: connect: Connection refused (error 111)
06:20:08.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:08.937489 3256 leader_election.cc:334] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 8a628d3d004b41b8beea7b87db236525 (127.2.224.125:37335): Network error: Client connection negotiation failed: client connection to 127.2.224.125:37335: connect: Connection refused (error 111)
06:20:08.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:08.937793 3256 leader_election.cc:302] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 8c99d7a808bd4466be4b84bb9e1cdadd; no voters: 8726f6ab2f024c2ea9d0e2c1ae24b2cd, 8a628d3d004b41b8beea7b87db236525
06:20:08.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:08.938470 3496 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
06:20:09.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:09.901954 3319 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:20:09.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:09.902308 3319 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:20:10.360 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.224.125:37335(127.2.224.125:37335)] connecting to peer
06:20:10.362 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.224.126:38429(127.2.224.126:38429)] connecting to peer
06:20:10.362 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:10.362 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.224.125:37335(127.2.224.125:37335)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:10.363 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.224.125:37335: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:10.364 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:10.364 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.224.126:38429(127.2.224.126:38429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:10.365 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.224.126:38429: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:10.365 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.2.224.124:35387,127.2.224.125:37335,127.2.224.126:38429; will retry
06:20:10.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:10.906682 3319 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:20:10.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:10.907397 3319 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:20:11.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:11.905441 2790 debug-util.cc:398] Leaking SignalData structure 0x7b08000af9a0 after lost signal to thread 2664
06:20:11.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:11.906503 2790 debug-util.cc:398] Leaking SignalData structure 0x7b08000b61a0 after lost signal to thread 2793
06:20:11.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:11.910813 3319 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:20:11.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:11.911088 3319 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:20:11.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:11.913806 3233 master_runner.cc:186] Error getting master registration for 127.2.224.126:38429: Network error: Client connection negotiation failed: client connection to 127.2.224.126:38429: connect: Connection refused (error 111),
06:20:11.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:11.914264 3233 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:20:12.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:12.913825 3319 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:20:12.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:12.914068 3319 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:20:13.820 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.224.125:37335(127.2.224.125:37335)] connecting to peer
06:20:13.821 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.224.126:38429(127.2.224.126:38429)] connecting to peer
06:20:13.822 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:13.823 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.224.125:37335(127.2.224.125:37335)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:13.823 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.224.125:37335: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:13.824 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:13.824 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.224.126:38429(127.2.224.126:38429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:13.824 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.224.126:38429: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:13.824 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.2.224.124:35387,127.2.224.125:37335,127.2.224.126:38429; will retry
06:20:13.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:13.916623 3319 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:20:13.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:13.916855 3319 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:20:14.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:14.919518 3319 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:20:14.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:14.919939 3319 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:20:15.800 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.224.125:37335(127.2.224.125:37335)] connecting to peer
06:20:15.801 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.224.126:38429(127.2.224.126:38429)] connecting to peer
06:20:15.803 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:15.803 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.224.125:37335(127.2.224.125:37335)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:15.803 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.224.125:37335: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:15.805 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:15.805 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.224.126:38429(127.2.224.126:38429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:15.805 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.224.126:38429: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:15.805 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.2.224.124:35387,127.2.224.125:37335,127.2.224.126:38429; will retry
06:20:15.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:15.923074 3319 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:20:15.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:15.923404 3319 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:20:16.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 06:20:16 dist-test-slave-0tdn krb5kdc[2957](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790490016, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.224.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
06:20:16.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:16.612411 3248 init.cc:295] Successfully reacquired a new kerberos TGT
06:20:16.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:16.917222 3233 master_runner.cc:186] Error getting master registration for 127.2.224.126:38429: Network error: Client connection negotiation failed: client connection to 127.2.224.126:38429: connect: Connection refused (error 111),
06:20:16.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:16.917719 3233 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:20:16.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:16.926056 3319 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:20:16.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:16.926280 3319 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:20:17.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:17.929111 3319 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:20:17.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:17.929374 3319 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:20:18.300 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.224.125:37335(127.2.224.125:37335)] connecting to peer
06:20:18.303 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.224.126:38429(127.2.224.126:38429)] connecting to peer
06:20:18.303 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:18.304 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.224.125:37335(127.2.224.125:37335)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:18.304 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.224.125:37335: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:18.305 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:18.305 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.224.126:38429(127.2.224.126:38429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:18.306 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.224.126:38429: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:18.306 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.2.224.124:35387,127.2.224.125:37335,127.2.224.126:38429; will retry
06:20:18.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:18.931968 3319 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:20:18.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:18.932307 3319 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:20:19.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:19.935097 3319 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:20:19.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:19.935328 3319 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:20:20.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:20.231909 3511 raft_consensus.cc:482] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:20:20.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:20.232318 3511 raft_consensus.cc:504] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8726f6ab2f024c2ea9d0e2c1ae24b2cd" member_type: VOTER last_known_addr { host: "127.2.224.126" port: 38429 } } peers { permanent_uuid: "8a628d3d004b41b8beea7b87db236525" member_type: VOTER last_known_addr { host: "127.2.224.125" port: 37335 } } peers { permanent_uuid: "8c99d7a808bd4466be4b84bb9e1cdadd" member_type: VOTER last_known_addr { host: "127.2.224.124" port: 35387 } }
06:20:20.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:20.233985 3511 leader_election.cc:288] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8726f6ab2f024c2ea9d0e2c1ae24b2cd (127.2.224.126:38429), 8a628d3d004b41b8beea7b87db236525 (127.2.224.125:37335)
06:20:20.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:20.236938 3254 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.224.126:38429: connect: Connection refused (error 111) [suppressed 1 similar messages]
06:20:20.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:20.239713 3254 leader_election.cc:334] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 8726f6ab2f024c2ea9d0e2c1ae24b2cd (127.2.224.126:38429): Network error: Client connection negotiation failed: client connection to 127.2.224.126:38429: connect: Connection refused (error 111)
06:20:20.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:20.240258 3256 leader_election.cc:334] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 8a628d3d004b41b8beea7b87db236525 (127.2.224.125:37335): Network error: Client connection negotiation failed: client connection to 127.2.224.125:37335: connect: Connection refused (error 111)
06:20:20.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:20.240532 3256 leader_election.cc:302] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 8c99d7a808bd4466be4b84bb9e1cdadd; no voters: 8726f6ab2f024c2ea9d0e2c1ae24b2cd, 8a628d3d004b41b8beea7b87db236525
06:20:20.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:20.241117 3511 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
06:20:20.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:20.938426 3319 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:20:20.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:20.938704 3319 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:20:21.400 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.224.125:37335(127.2.224.125:37335)] connecting to peer
06:20:21.401 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.224.126:38429(127.2.224.126:38429)] connecting to peer
06:20:21.403 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:21.403 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.224.125:37335(127.2.224.125:37335)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:21.403 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.224.125:37335: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:21.405 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:21.405 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.224.126:38429(127.2.224.126:38429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:21.405 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.224.126:38429: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:21.405 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.2.224.124:35387,127.2.224.125:37335,127.2.224.126:38429; will retry
06:20:21.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:21.920008 3233 master_runner.cc:186] Error getting master registration for 127.2.224.126:38429: Network error: Client connection negotiation failed: client connection to 127.2.224.126:38429: connect: Connection refused (error 111),
06:20:21.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:21.920372 3233 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:20:21.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:21.941648 3319 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:20:21.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:21.941956 3319 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:20:22.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:22.944619 3319 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:20:22.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:22.944883 3319 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:20:23.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:23.947620 3319 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:20:23.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:23.947913 3319 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:20:24.200 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.224.125:37335(127.2.224.125:37335)] connecting to peer
06:20:24.201 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.224.126:38429(127.2.224.126:38429)] connecting to peer
06:20:24.202 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:24.202 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:24.202 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.224.125:37335(127.2.224.125:37335)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:24.203 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.224.126:38429(127.2.224.126:38429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:24.203 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.224.125:37335: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:24.203 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.224.126:38429: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:24.204 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.2.224.124:35387,127.2.224.125:37335,127.2.224.126:38429; will retry
06:20:24.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:24.950714 3319 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:20:24.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:24.950974 3319 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:20:25.920 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.224.125:37335(127.2.224.125:37335)] connecting to peer
06:20:25.922 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:25.923 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.224.125:37335(127.2.224.125:37335)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:25.923 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.224.126:38429(127.2.224.126:38429)] connecting to peer
06:20:25.923 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.224.125:37335: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:25.924 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:25.924 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.224.126:38429(127.2.224.126:38429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:25.924 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.224.126:38429: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:25.925 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.2.224.124:35387,127.2.224.125:37335,127.2.224.126:38429; will retry
06:20:25.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:25.954135 3319 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:20:25.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:25.954450 3319 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:20:26.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 06:20:26 dist-test-slave-0tdn krb5kdc[2957](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790490026, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.224.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
06:20:26.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:26.614881 3248 init.cc:295] Successfully reacquired a new kerberos TGT
06:20:26.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:26.923260 3233 master_runner.cc:186] Error getting master registration for 127.2.224.126:38429: Network error: Client connection negotiation failed: client connection to 127.2.224.126:38429: connect: Connection refused (error 111),
06:20:26.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:26.923661 3233 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:20:26.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:26.956997 3319 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:20:26.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:26.957305 3319 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:20:27.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:27.959686 3319 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:20:27.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:27.960019 3319 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:20:28.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:28.929910 3528 raft_consensus.cc:482] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:20:28.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:28.930341 3528 raft_consensus.cc:504] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8726f6ab2f024c2ea9d0e2c1ae24b2cd" member_type: VOTER last_known_addr { host: "127.2.224.126" port: 38429 } } peers { permanent_uuid: "8a628d3d004b41b8beea7b87db236525" member_type: VOTER last_known_addr { host: "127.2.224.125" port: 37335 } } peers { permanent_uuid: "8c99d7a808bd4466be4b84bb9e1cdadd" member_type: VOTER last_known_addr { host: "127.2.224.124" port: 35387 } }
06:20:28.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:28.932017 3528 leader_election.cc:288] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8726f6ab2f024c2ea9d0e2c1ae24b2cd (127.2.224.126:38429), 8a628d3d004b41b8beea7b87db236525 (127.2.224.125:37335)
06:20:28.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:28.933919 3254 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.224.126:38429: connect: Connection refused (error 111) [suppressed 1 similar messages]
06:20:28.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:28.937542 3254 leader_election.cc:334] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 8726f6ab2f024c2ea9d0e2c1ae24b2cd (127.2.224.126:38429): Network error: Client connection negotiation failed: client connection to 127.2.224.126:38429: connect: Connection refused (error 111)
06:20:28.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:28.938023 3256 leader_election.cc:334] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 8a628d3d004b41b8beea7b87db236525 (127.2.224.125:37335): Network error: Client connection negotiation failed: client connection to 127.2.224.125:37335: connect: Connection refused (error 111)
06:20:28.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:28.938295 3256 leader_election.cc:302] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 8c99d7a808bd4466be4b84bb9e1cdadd; no voters: 8726f6ab2f024c2ea9d0e2c1ae24b2cd, 8a628d3d004b41b8beea7b87db236525
06:20:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:28.938854 3528 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
06:20:28.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:28.962395 3319 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:20:28.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:28.962613 3319 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:20:29.500 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.224.125:37335(127.2.224.125:37335)] connecting to peer
06:20:29.501 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.224.126:38429(127.2.224.126:38429)] connecting to peer
06:20:29.501 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:29.501 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.224.125:37335(127.2.224.125:37335)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:29.501 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.224.125:37335: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:29.503 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:29.503 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.224.126:38429(127.2.224.126:38429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:29.503 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.224.126:38429: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:29.504 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.2.224.124:35387,127.2.224.125:37335,127.2.224.126:38429; will retry
06:20:29.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:29.965090 3319 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8c99d7a808bd4466be4b84bb9e1cdadd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:20:29.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:29.965401 3319 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:20:30.340 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.224.125:37335(127.2.224.125:37335)] connecting to peer
06:20:30.340 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.224.126:38429(127.2.224.126:38429)] connecting to peer
06:20:30.341 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:30.341 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.224.125:37335(127.2.224.125:37335)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:30.341 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.224.125:37335: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:30.341 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:30.341 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.224.126:38429(127.2.224.126:38429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:30.342 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.224.126:38429: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:30.343 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.2.224.124:35387,127.2.224.125:37335,127.2.224.126:38429; will retry
06:20:30.439 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.224.125:37335(127.2.224.125:37335)] connecting to peer
06:20:30.440 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.224.126:38429(127.2.224.126:38429)] connecting to peer
06:20:30.440 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:30.441 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.224.125:37335(127.2.224.125:37335)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:30.441 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.224.125:37335: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335
06:20:30.441 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:30.441 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.224.126:38429(127.2.224.126:38429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:30.441 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.224.126:38429: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
06:20:30.443 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.2.224.124:35387,127.2.224.125:37335,127.2.224.126:38429; will retry
06:20:30.446 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=23, TimeoutTracker(timeout=30000, elapsed=29605), Traces: [0ms] refreshing cache from master, [50ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.124:35387, [136ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.125:37335, [137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.126:38429, [197ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.125:37335: Network error: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335, [247ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.126:38429: Network error: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [506ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.124:35387: OK, [510ms] delaying RPC due to: Service unavailable: Master config (127.2.224.124:35387,127.2.224.125:37335,127.2.224.126:38429) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [515ms] refreshing cache from master, [515ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.124:35387, [517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.125:37335, [520ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.126:38429, [521ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.125:37335: Network error: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335, [522ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.124:35387: OK, [525ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.126:38429: Network error: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [526ms] delaying RPC due to: Service unavailable: Master config (127.2.224.124:35387,127.2.224.125:37335,127.2.224.126:38429) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [535ms] refreshing cache from master, [535ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.124:35387, [536ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.125:37335, [538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.126:38429, [539ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.125:37335: Network error: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335, [541ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.124:35387: OK, [542ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.126:38429: Network error: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [544ms] delaying RPC due to: Service unavailable: Master config (127.2.224.124:35387,127.2.224.125:37335,127.2.224.126:38429) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [555ms] refreshing cache from master, [555ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.124:35387, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.125:37335, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.126:38429, [560ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.125:37335: Network error: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335, [561ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.124:35387: OK, [562ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.126:38429: Network error: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [565ms] delaying RPC due to: Service unavailable: Master config (127.2.224.124:35387,127.2.224.125:37335,127.2.224.126:38429) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [575ms] refreshing cache from master, [575ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.124:35387, [575ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.125:37335, [579ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.125:37335: Network error: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335, [580ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.124:35387: OK, [580ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.126:38429, [582ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.126:38429: Network error: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [583ms] delaying RPC due to: Service unavailable: Master config (127.2.224.124:35387,127.2.224.125:37335,127.2.224.126:38429) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [615ms] refreshing cache from master, [615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.124:35387, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.125:37335, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.126:38429, [620ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.124:35387: OK, [620ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.125:37335: Network error: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335, [624ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.126:38429: Network error: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [625ms] delaying RPC due to: Service unavailable: Master config (127.2.224.124:35387,127.2.224.125:37335,127.2.224.126:38429) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [695ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.124:35387, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.125:37335, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.126:38429, [699ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.124:35387: OK, [700ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.125:37335: Network error: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335, [701ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.126:38429: Network error: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [702ms] delaying RPC due to: Service unavailable: Master config (127.2.224.124:35387,127.2.224.125:37335,127.2.224.126:38429) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [815ms] refreshing cache from master, [816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.124:35387, [816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.125:37335, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.126:38429, [820ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.125:37335: Network error: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335, [821ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.124:35387: OK, [822ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.126:38429: Network error: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [823ms] delaying RPC due to: Service unavailable: Master config (127.2.224.124:35387,127.2.224.125:37335,127.2.224.126:38429) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [1015ms] refreshing cache from master, [1016ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.124:35387, [1016ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.125:37335, [1018ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.126:38429, [1018ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.125:37335: Network error: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335, [1021ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.124:35387: OK, [1022ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.126:38429: Network error: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [1023ms] delaying RPC due to: Service unavailable: Master config (127.2.224.124:35387,127.2.224.125:37335,127.2.224.126:38429) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [1135ms] refreshing cache from master, [1135ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.124:35387, [1135ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.125:37335, [1137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.126:38429, [1138ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.125:37335: Network error: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335, [1139ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.124:35387: OK, [1141ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.126:38429: Network error: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [1142ms] delaying RPC due to: Service unavailable: Master config (127.2.224.124:35387,127.2.224.125:37335,127.2.224.126:38429) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [1255ms] refreshing cache from master, [1256ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.124:35387, [1256ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.125:37335, [1257ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.126:38429, [1259ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.125:37335: Network error: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335, [1259ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.126:38429: Network error: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [1260ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.124:35387: OK, [1260ms] delaying RPC due to: Service unavailable: Master config (127.2.224.124:35387,127.2.224.125:37335,127.2.224.126:38429) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [2075ms] refreshing cache from master, [2075ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.124:35387, [2076ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.125:37335, [2077ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.126:38429, [2079ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.125:37335: Network error: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335, [2080ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.124:35387: OK, [2080ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.126:38429: Network error: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [2081ms] delaying RPC due to: Service unavailable: Master config (127.2.224.124:35387,127.2.224.125:37335,127.2.224.126:38429) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [6135ms] refreshing cache from master, [6135ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.124:35387, [6136ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.125:37335, [6136ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.126:38429, [6138ms] too many traces: truncated at 100 traces, deferred=Deferred@831431041(state=PENDING, result=null, callback=(continuation of Deferred@1650894087 after retry RPC after error@1838159039) -> (continuation of Deferred@511984230 after retry RPC after error@1514302110) -> (continuation of Deferred@224113228 after retry RPC after error@1619225268) -> (continuation of Deferred@1546163120 after retry RPC after error@360121393) -> (continuation of Deferred@1164840193 after retry RPC after error@1044584914) -> (continuation of Deferred@644786361 after retry RPC after error@185956041) -> (continuation of Deferred@1042707631 after retry RPC after error@403535664) -> (continuation of Deferred@1265971176 after retry RPC after error@1802939249) -> (continuation of Deferred@1727539644 after retry RPC after error@364865188) -> (continuation of Deferred@2010583444 after retry RPC after error@690225468) -> (continuation of Deferred@238450797 after retry RPC after error@1624417609) -> (continuation of Deferred@801526646 after retry RPC after error@652380755) -> (continuation of Deferred@455769074 after retry RPC after error@1856470323) -> (continuation of Deferred@1936186994 after retry RPC after error@1097167455) -> (continuation of Deferred@156791914 after retry RPC after error@1556340375) -> (continuation of Deferred@286924595 after retry RPC after error@2064085189) -> (continuation of Deferred@825663723 after retry RPC after error@1942218556) -> (continuation of Deferred@845254128 after retry RPC after error@1923109409) -> (continuation of Deferred@1989514598 after retry RPC after error@436430479) -> (continuation of Deferred@607348274 after retry RPC after error@1416759759) -> (continuation of Deferred@244750466 after retry RPC after error@262185153) -> (continuation of Deferred@807224817 after retry RPC after error@1961168460), errback=(continuation of Deferred@1650894087 after retry RPC after error@1838159039) -> (continuation of Deferred@511984230 after retry RPC after error@1514302110) -> (continuation of Deferred@224113228 after retry RPC after error@1619225268) -> (continuation of Deferred@1546163120 after retry RPC after error@360121393) -> (continuation of Deferred@1164840193 after retry RPC after error@1044584914) -> (continuation of Deferred@644786361 after retry RPC after error@185956041) -> (continuation of Deferred@1042707631 after retry RPC after error@403535664) -> (continuation of Deferred@1265971176 after retry RPC after error@1802939249) -> (continuation of Deferred@1727539644 after retry RPC after error@364865188) -> (continuation of Deferred@2010583444 after retry RPC after error@690225468) -> (continuation of Deferred@238450797 after retry RPC after error@1624417609) -> (continuation of Deferred@801526646 after retry RPC after error@652380755) -> (continuation of Deferred@455769074 after retry RPC after error@1856470323) -> (continuation of Deferred@1936186994 after retry RPC after error@1097167455) -> (continuation of Deferred@156791914 after retry RPC after error@1556340375) -> (continuation of Deferred@286924595 after retry RPC after error@2064085189) -> (continuation of Deferred@825663723 after retry RPC after error@1942218556) -> (continuation of Deferred@845254128 after retry RPC after error@1923109409) -> (continuation of Deferred@1989514598 after retry RPC after error@436430479) -> (continuation of Deferred@607348274 after retry RPC after error@1416759759) -> (continuation of Deferred@244750466 after retry RPC after error@262185153) -> (continuation of Deferred@807224817 after retry RPC after error@1961168460)))
06:20:30.450 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.2.224.124:35387(127.2.224.124:35387)] disconnecting while in state READY
06:20:30.451 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.2.224.125:37335(127.2.224.125:37335)] disconnecting while in state TERMINATED
06:20:30.451 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.2.224.126:38429(127.2.224.126:38429)] disconnecting while in state TERMINATED
06:20:30.456 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
06:20:30.459 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
06:20:30.459 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.2.224.124:35387(127.2.224.124:35387)] handling channelInactive
06:20:30.466 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.224.124:35387(127.2.224.124:35387)] cleaning up while in state READY due to: connection closed
06:20:30.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:30.466027 2945 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskwUNst7/build/tsan/bin/kudu with pid 3233
06:20:31.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T06:20:31Z chronyd exiting
06:20:32.665 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
06:20:32.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:32.667215 2220 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskwUNst7/build/tsan/bin/kudu with pid 2499
06:20:32.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:32.695497 2220 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskwUNst7/build/tsan/bin/kudu with pid 2657
06:20:32.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:32.719396 2220 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskwUNst7/build/tsan/bin/kudu with pid 2803
06:20:32.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:32.743917 2220 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskwUNst7/build/tsan/bin/kudu with pid 2232
06:20:32.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:32.772688 2403 connection.cc:535] client connection to 127.2.43.62:41039 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
06:20:32.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:32.773204 2403 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
06:20:32.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:32.774049 2220 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskwUNst7/build/tsan/bin/kudu with pid 2302
06:20:32.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 06:20:32.778209 2403 consensus_peers.cc:487] T 00000000000000000000000000000000 P 0c1c70b81b304462b7e21f739ac90081 -> Peer 19fa63f7ad554a74877915078159d1f5 (127.2.43.62:41039): Couldn't send request to peer 19fa63f7ad554a74877915078159d1f5. Status: Network error: Client connection negotiation failed: client connection to 127.2.43.62:41039: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
06:20:32.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 06:20:32.804101 2220 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskwUNst7/build/tsan/bin/kudu with pid 2377
06:20:33.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T06:20:33Z chronyd exiting
06:20:34.957 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=23, TimeoutTracker(timeout=30000, elapsed=29605), Traces: [0ms] refreshing cache from master, [50ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.124:35387, [136ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.125:37335, [137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.126:38429, [197ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.125:37335: Network error: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335, [247ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.126:38429: Network error: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [506ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.124:35387: OK, [510ms] delaying RPC due to: Service unavailable: Master config (127.2.224.124:35387,127.2.224.125:37335,127.2.224.126:38429) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [515ms] refreshing cache from master, [515ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.124:35387, [517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.125:37335, [520ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.126:38429, [521ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.125:37335: Network error: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335, [522ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.124:35387: OK, [525ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.126:38429: Network error: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [526ms] delaying RPC due to: Service unavailable: Master config (127.2.224.124:35387,127.2.224.125:37335,127.2.224.126:38429) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [535ms] refreshing cache from master, [535ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.124:35387, [536ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.125:37335, [538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.126:38429, [539ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.125:37335: Network error: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335, [541ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.124:35387: OK, [542ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.126:38429: Network error: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [544ms] delaying RPC due to: Service unavailable: Master config (127.2.224.124:35387,127.2.224.125:37335,127.2.224.126:38429) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [555ms] refreshing cache from master, [555ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.124:35387, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.125:37335, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.126:38429, [560ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.125:37335: Network error: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335, [561ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.124:35387: OK, [562ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.126:38429: Network error: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [565ms] delaying RPC due to: Service unavailable: Master config (127.2.224.124:35387,127.2.224.125:37335,127.2.224.126:38429) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [575ms] refreshing cache from master, [575ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.124:35387, [575ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.125:37335, [579ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.125:37335: Network error: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335, [580ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.124:35387: OK, [580ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.126:38429, [582ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.126:38429: Network error: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [583ms] delaying RPC due to: Service unavailable: Master config (127.2.224.124:35387,127.2.224.125:37335,127.2.224.126:38429) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [615ms] refreshing cache from master, [615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.124:35387, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.125:37335, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.126:38429, [620ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.124:35387: OK, [620ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.125:37335: Network error: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335, [624ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.126:38429: Network error: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [625ms] delaying RPC due to: Service unavailable: Master config (127.2.224.124:35387,127.2.224.125:37335,127.2.224.126:38429) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [695ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.124:35387, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.125:37335, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.126:38429, [699ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.124:35387: OK, [700ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.125:37335: Network error: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335, [701ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.126:38429: Network error: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [702ms] delaying RPC due to: Service unavailable: Master config (127.2.224.124:35387,127.2.224.125:37335,127.2.224.126:38429) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [815ms] refreshing cache from master, [816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.124:35387, [816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.125:37335, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.126:38429, [820ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.125:37335: Network error: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335, [821ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.124:35387: OK, [822ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.126:38429: Network error: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [823ms] delaying RPC due to: Service unavailable: Master config (127.2.224.124:35387,127.2.224.125:37335,127.2.224.126:38429) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [1015ms] refreshing cache from master, [1016ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.124:35387, [1016ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.125:37335, [1018ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.126:38429, [1018ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.125:37335: Network error: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335, [1021ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.124:35387: OK, [1022ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.126:38429: Network error: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [1023ms] delaying RPC due to: Service unavailable: Master config (127.2.224.124:35387,127.2.224.125:37335,127.2.224.126:38429) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [1135ms] refreshing cache from master, [1135ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.124:35387, [1135ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.125:37335, [1137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.126:38429, [1138ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.125:37335: Network error: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335, [1139ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.124:35387: OK, [1141ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.126:38429: Network error: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [1142ms] delaying RPC due to: Service unavailable: Master config (127.2.224.124:35387,127.2.224.125:37335,127.2.224.126:38429) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [1255ms] refreshing cache from master, [1256ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.124:35387, [1256ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.125:37335, [1257ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.126:38429, [1259ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.125:37335: Network error: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335, [1259ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.126:38429: Network error: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [1260ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.124:35387: OK, [1260ms] delaying RPC due to: Service unavailable: Master config (127.2.224.124:35387,127.2.224.125:37335,127.2.224.126:38429) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [2075ms] refreshing cache from master, [2075ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.124:35387, [2076ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.125:37335, [2077ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.126:38429, [2079ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.125:37335: Network error: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335, [2080ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.124:35387: OK, [2080ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.126:38429: Network error: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [2081ms] delaying RPC due to: Service unavailable: Master config (127.2.224.124:35387,127.2.224.125:37335,127.2.224.126:38429) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [6135ms] refreshing cache from master, [6135ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.124:35387, [6136ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.125:37335, [6136ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.126:38429, [6138ms] too many traces: truncated at 100 traces, deferred=Deferred@831431041(state=PENDING, result=null, callback=(continuation of Deferred@1650894087 after retry RPC after error@1838159039) -> (continuation of Deferred@511984230 after retry RPC after error@1514302110) -> (continuation of Deferred@224113228 after retry RPC after error@1619225268) -> (continuation of Deferred@1546163120 after retry RPC after error@360121393) -> (continuation of Deferred@1164840193 after retry RPC after error@1044584914) -> (continuation of Deferred@644786361 after retry RPC after error@185956041) -> (continuation of Deferred@1042707631 after retry RPC after error@403535664) -> (continuation of Deferred@1265971176 after retry RPC after error@1802939249) -> (continuation of Deferred@1727539644 after retry RPC after error@364865188) -> (continuation of Deferred@2010583444 after retry RPC after error@690225468) -> (continuation of Deferred@238450797 after retry RPC after error@1624417609) -> (continuation of Deferred@801526646 after retry RPC after error@652380755) -> (continuation of Deferred@455769074 after retry RPC after error@1856470323) -> (continuation of Deferred@1936186994 after retry RPC after error@1097167455) -> (continuation of Deferred@156791914 after retry RPC after error@1556340375) -> (continuation of Deferred@286924595 after retry RPC after error@2064085189) -> (continuation of Deferred@825663723 after retry RPC after error@1942218556) -> (continuation of Deferred@845254128 after retry RPC after error@1923109409) -> (continuation of Deferred@1989514598 after retry RPC after error@436430479) -> (continuation of Deferred@607348274 after retry RPC after error@1416759759) -> (continuation of Deferred@244750466 after retry RPC after error@262185153) -> (continuation of Deferred@807224817 after retry RPC after error@1961168460), errback=(continuation of Deferred@1650894087 after retry RPC after error@1838159039) -> (continuation of Deferred@511984230 after retry RPC after error@1514302110) -> (continuation of Deferred@224113228 after retry RPC after error@1619225268) -> (continuation of Deferred@1546163120 after retry RPC after error@360121393) -> (continuation of Deferred@1164840193 after retry RPC after error@1044584914) -> (continuation of Deferred@644786361 after retry RPC after error@185956041) -> (continuation of Deferred@1042707631 after retry RPC after error@403535664) -> (continuation of Deferred@1265971176 after retry RPC after error@1802939249) -> (continuation of Deferred@1727539644 after retry RPC after error@364865188) -> (continuation of Deferred@2010583444 after retry RPC after error@690225468) -> (continuation of Deferred@238450797 after retry RPC after error@1624417609) -> (continuation of Deferred@801526646 after retry RPC after error@652380755) -> (continuation of Deferred@455769074 after retry RPC after error@1856470323) -> (continuation of Deferred@1936186994 after retry RPC after error@1097167455) -> (continuation of Deferred@156791914 after retry RPC after error@1556340375) -> (continuation of Deferred@286924595 after retry RPC after error@2064085189) -> (continuation of Deferred@825663723 after retry RPC after error@1942218556) -> (continuation of Deferred@845254128 after retry RPC after error@1923109409) -> (continuation of Deferred@1989514598 after retry RPC after error@436430479) -> (continuation of Deferred@607348274 after retry RPC after error@1416759759) -> (continuation of Deferred@244750466 after retry RPC after error@262185153) -> (continuation of Deferred@807224817 after retry RPC after error@1961168460)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.2.224.124:35387,127.2.224.125:37335,127.2.224.126:38429) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.2.224.126:38429
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=23, TimeoutTracker(timeout=30000, elapsed=29605), Traces: [0ms] refreshing cache from master, [50ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.124:35387, [136ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.125:37335, [137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.126:38429, [197ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.125:37335: Network error: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335, [247ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.126:38429: Network error: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [506ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.124:35387: OK, [510ms] delaying RPC due to: Service unavailable: Master config (127.2.224.124:35387,127.2.224.125:37335,127.2.224.126:38429) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [515ms] refreshing cache from master, [515ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.124:35387, [517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.125:37335, [520ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.126:38429, [521ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.125:37335: Network error: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335, [522ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.124:35387: OK, [525ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.126:38429: Network error: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [526ms] delaying RPC due to: Service unavailable: Master config (127.2.224.124:35387,127.2.224.125:37335,127.2.224.126:38429) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [535ms] refreshing cache from master, [535ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.124:35387, [536ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.125:37335, [538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.126:38429, [539ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.125:37335: Network error: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335, [541ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.124:35387: OK, [542ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.126:38429: Network error: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [544ms] delaying RPC due to: Service unavailable: Master config (127.2.224.124:35387,127.2.224.125:37335,127.2.224.126:38429) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [555ms] refreshing cache from master, [555ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.124:35387, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.125:37335, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.126:38429, [560ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.125:37335: Network error: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335, [561ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.124:35387: OK, [562ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.126:38429: Network error: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [565ms] delaying RPC due to: Service unavailable: Master config (127.2.224.124:35387,127.2.224.125:37335,127.2.224.126:38429) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [575ms] refreshing cache from master, [575ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.124:35387, [575ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.125:37335, [579ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.125:37335: Network error: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335, [580ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.124:35387: OK, [580ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.126:38429, [582ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.126:38429: Network error: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [583ms] delaying RPC due to: Service unavailable: Master config (127.2.224.124:35387,127.2.224.125:37335,127.2.224.126:38429) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [615ms] refreshing cache from master, [615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.124:35387, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.125:37335, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.126:38429, [620ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.124:35387: OK, [620ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.125:37335: Network error: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335, [624ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.126:38429: Network error: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [625ms] delaying RPC due to: Service unavailable: Master config (127.2.224.124:35387,127.2.224.125:37335,127.2.224.126:38429) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [695ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.124:35387, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.125:37335, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.126:38429, [699ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.124:35387: OK, [700ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.125:37335: Network error: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335, [701ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.126:38429: Network error: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [702ms] delaying RPC due to: Service unavailable: Master config (127.2.224.124:35387,127.2.224.125:37335,127.2.224.126:38429) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [815ms] refreshing cache from master, [816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.124:35387, [816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.125:37335, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.126:38429, [820ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.125:37335: Network error: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335, [821ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.124:35387: OK, [822ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.126:38429: Network error: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [823ms] delaying RPC due to: Service unavailable: Master config (127.2.224.124:35387,127.2.224.125:37335,127.2.224.126:38429) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [1015ms] refreshing cache from master, [1016ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.124:35387, [1016ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.125:37335, [1018ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.126:38429, [1018ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.125:37335: Network error: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335, [1021ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.124:35387: OK, [1022ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.126:38429: Network error: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [1023ms] delaying RPC due to: Service unavailable: Master config (127.2.224.124:35387,127.2.224.125:37335,127.2.224.126:38429) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [1135ms] refreshing cache from master, [1135ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.124:35387, [1135ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.125:37335, [1137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.126:38429, [1138ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.125:37335: Network error: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335, [1139ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.124:35387: OK, [1141ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.126:38429: Network error: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [1142ms] delaying RPC due to: Service unavailable: Master config (127.2.224.124:35387,127.2.224.125:37335,127.2.224.126:38429) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [1255ms] refreshing cache from master, [1256ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.124:35387, [1256ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.125:37335, [1257ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.126:38429, [1259ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.125:37335: Network error: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335, [1259ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.126:38429: Network error: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [1260ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.124:35387: OK, [1260ms] delaying RPC due to: Service unavailable: Master config (127.2.224.124:35387,127.2.224.125:37335,127.2.224.126:38429) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [2075ms] refreshing cache from master, [2075ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.124:35387, [2076ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.125:37335, [2077ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.126:38429, [2079ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.125:37335: Network error: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335, [2080ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.124:35387: OK, [2080ms] Sub RPC ConnectToMaster: received response from server master-127.2.224.126:38429: Network error: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [2081ms] delaying RPC due to: Service unavailable: Master config (127.2.224.124:35387,127.2.224.125:37335,127.2.224.126:38429) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429, [6135ms] refreshing cache from master, [6135ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.124:35387, [6136ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.125:37335, [6136ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.224.126:38429, [6138ms] too many traces: truncated at 100 traces, deferred=Deferred@831431041(state=PENDING, result=null, callback=(continuation of Deferred@1650894087 after retry RPC after error@1838159039) -> (continuation of Deferred@511984230 after retry RPC after error@1514302110) -> (continuation of Deferred@224113228 after retry RPC after error@1619225268) -> (continuation of Deferred@1546163120 after retry RPC after error@360121393) -> (continuation of Deferred@1164840193 after retry RPC after error@1044584914) -> (continuation of Deferred@644786361 after retry RPC after error@185956041) -> (continuation of Deferred@1042707631 after retry RPC after error@403535664) -> (continuation of Deferred@1265971176 after retry RPC after error@1802939249) -> (continuation of Deferred@1727539644 after retry RPC after error@364865188) -> (continuation of Deferred@2010583444 after retry RPC after error@690225468) -> (continuation of Deferred@238450797 after retry RPC after error@1624417609) -> (continuation of Deferred@801526646 after retry RPC after error@652380755) -> (continuation of Deferred@455769074 after retry RPC after error@1856470323) -> (continuation of Deferred@1936186994 after retry RPC after error@1097167455) -> (continuation of Deferred@156791914 after retry RPC after error@1556340375) -> (continuation of Deferred@286924595 after retry RPC after error@2064085189) -> (continuation of Deferred@825663723 after retry RPC after error@1942218556) -> (continuation of Deferred@845254128 after retry RPC after error@1923109409) -> (continuation of Deferred@1989514598 after retry RPC after error@436430479) -> (continuation of Deferred@607348274 after retry RPC after error@1416759759) -> (continuation of Deferred@244750466 after retry RPC after error@262185153) -> (continuation of Deferred@807224817 after retry RPC after error@1961168460), errback=(continuation of Deferred@1650894087 after retry RPC after error@1838159039) -> (continuation of Deferred@511984230 after retry RPC after error@1514302110) -> (continuation of Deferred@224113228 after retry RPC after error@1619225268) -> (continuation of Deferred@1546163120 after retry RPC after error@360121393) -> (continuation of Deferred@1164840193 after retry RPC after error@1044584914) -> (continuation of Deferred@644786361 after retry RPC after error@185956041) -> (continuation of Deferred@1042707631 after retry RPC after error@403535664) -> (continuation of Deferred@1265971176 after retry RPC after error@1802939249) -> (continuation of Deferred@1727539644 after retry RPC after error@364865188) -> (continuation of Deferred@2010583444 after retry RPC after error@690225468) -> (continuation of Deferred@238450797 after retry RPC after error@1624417609) -> (continuation of Deferred@801526646 after retry RPC after error@652380755) -> (continuation of Deferred@455769074 after retry RPC after error@1856470323) -> (continuation of Deferred@1936186994 after retry RPC after error@1097167455) -> (continuation of Deferred@156791914 after retry RPC after error@1556340375) -> (continuation of Deferred@286924595 after retry RPC after error@2064085189) -> (continuation of Deferred@825663723 after retry RPC after error@1942218556) -> (continuation of Deferred@845254128 after retry RPC after error@1923109409) -> (continuation of Deferred@1989514598 after retry RPC after error@436430479) -> (continuation of Deferred@607348274 after retry RPC after error@1416759759) -> (continuation of Deferred@244750466 after retry RPC after error@262185153) -> (continuation of Deferred@807224817 after retry RPC after error@1961168460)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.2.224.124:35387,127.2.224.125:37335,127.2.224.126:38429) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.125:37335(127.2.224.125:37335): Connection refused: /127.2.224.125:37335,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.224.126:38429(127.2.224.126:38429): Connection refused: /127.2.224.126:38429
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.2.224.126:38429
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)